diff --git a/.gitmodules b/.gitmodules index e9af594edc..07e4bb2b9c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "deps/TSZ"] path = deps/TSZ url = https://github.com/taosdata/TSZ.git +[submodule "examples/rust"] + path = examples/rust + url = https://github.com/songtianyi/tdengine-rust-bindings.git diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c index b28adfaaca..12be3deb2e 100644 --- a/contrib/test/craft/raftMain.c +++ b/contrib/test/craft/raftMain.c @@ -377,7 +377,7 @@ void printConf(SRaftServerConfig *pConf) { int main(int argc, char **argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); int32_t ret; exe_name = argv[0]; diff --git a/contrib/test/craft/simulate_vnode.c b/contrib/test/craft/simulate_vnode.c index 668fe638b7..7ee9b9f8f0 100644 --- a/contrib/test/craft/simulate_vnode.c +++ b/contrib/test/craft/simulate_vnode.c @@ -132,7 +132,7 @@ static void proposeValue(struct raft *r) { buf.base = raft_malloc(buf.len); // mock ts value - int vid = rand() % VNODE_COUNT; + int vid = taosRand() % VNODE_COUNT; snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL)); printf("propose value: %s \n", (char*)buf.base); @@ -174,7 +174,7 @@ void usage() { } int main(int argc, char **argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); exe_name = argv[0]; if (argc < 2) { diff --git a/examples/c/schemaless.c b/examples/c/schemaless.c index 21f39213cd..99aa361b0a 100644 --- a/examples/c/schemaless.c +++ b/examples/c/schemaless.c @@ -19,7 +19,7 @@ void shuffle(char**lines, size_t n) size_t i; for (i = 0; i < n - 1; i++) { - size_t j = i + rand() / (RAND_MAX / (n - i) + 1); + size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1); char* t = lines[j]; lines[j] = lines[i]; lines[i] = t; diff --git a/examples/rust b/examples/rust new file mode 160000 index 0000000000..1c8924dc66 --- /dev/null +++ b/examples/rust @@ -0,0 +1 @@ +Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 69c2618ac8..99360b8d3f 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -56,6 +56,11 @@ typedef enum { TSDB_STATIS_NONE = 1, // statis part not exist } ETsdbStatisStatus; +typedef enum { + TSDB_SMA_STAT_OK = 0, // ready to provide service + TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired +} ETsdbSmaStat; + extern char *qtypeStr[]; #ifdef __cplusplus diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ea09c62819..b5bd088006 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -56,7 +56,7 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; - int32_t tupleSize; + int32_t rowSize; int32_t numOfCols; union {int64_t uid; int64_t blockId;}; } SDataBlockInfo; @@ -70,10 +70,10 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg* pBlockAgg; - SArray* pDataBlock; // SArray - SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg *pBlockAgg; + SArray *pDataBlock; // SArray + SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { @@ -136,7 +136,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) return (void*)buf; } -static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) { +static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); @@ -157,7 +157,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp return tlen; } -static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { +static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); @@ -195,7 +195,7 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { // tfree(pBlock); } -static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { +static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { if (pRsp->schemas) { if (pRsp->schemas->nCols) { tfree(pRsp->schemas->pSchema); @@ -218,18 +218,17 @@ typedef struct SColumn { int64_t dataBlockId; }; - char name[TSDB_COL_NAME_LEN]; - int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) union { int16_t colId; int16_t slotId; }; + char name[TSDB_COL_NAME_LEN]; + int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) int16_t type; int32_t bytes; uint8_t precision; uint8_t scale; - // SColumnInfo info; } SColumn; typedef struct SLimit { @@ -254,12 +253,20 @@ typedef struct SFunctParam { } SFunctParam; // the structure for sql function in select clause +typedef struct SResSchame { + int8_t type; + int32_t colId; + int32_t bytes; + int32_t precision; + int32_t scale; + char name[TSDB_COL_NAME_LEN]; +} SResSchema; + +// TODO move away to executor.h typedef struct SExprBasicInfo { - SSchema resSchema; // TODO refactor - int32_t interBytes; // inter result buffer size, TODO remove it + SResSchema resSchema; int16_t numOfParams; // argument value of each function SFunctParam *pParam; -// SVariant param[3]; // parameters are not more than 3 } SExprBasicInfo; typedef struct SExprInfo { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c114457b6b..c983f68be9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -777,7 +777,7 @@ typedef struct SVgroupInfo { int32_t vgId; uint32_t hashBegin; uint32_t hashEnd; - SEpSet epset; + SEpSet epSet; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SVgroupInfo; @@ -1904,8 +1904,8 @@ typedef struct { } SVCreateTSmaReq; typedef struct { - int8_t type; // 0 status report, 1 update data - char indexName[TSDB_INDEX_NAME_LEN + 1]; // + int8_t type; // 0 status report, 1 update data + char indexName[TSDB_INDEX_NAME_LEN + 1]; // STimeWindow windows; } STSmaMsg; @@ -2101,7 +2101,7 @@ typedef struct { int32_t skipLogNum; int32_t numOfTopics; SArray* pBlockData; // SArray -} SMqConsumeRsp; +} SMqPollRsp; // one req for one vg+topic typedef struct { @@ -2114,7 +2114,7 @@ typedef struct { int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; -} SMqConsumeReq; +} SMqPollReq; typedef struct { int32_t vgId; @@ -2136,7 +2136,7 @@ typedef struct { struct tmq_message_t { SMqRspHead head; union { - SMqConsumeRsp consumeRsp; + SMqPollRsp consumeRsp; SMqCMGetSubEpRsp getEpRsp; }; void* extra; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 21013ef906..c01e267c42 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -132,11 +132,11 @@ struct SqlFunctionCtx; struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified -typedef struct SExtTagsInfo { - int16_t tagsLen; // keep the tags data for top/bottom query result - int16_t numOfTagCols; - struct SqlFunctionCtx **pTagCtxList; -} SExtTagsInfo; +typedef struct SSubsidiaryResInfo { + int16_t bufLen; // keep the tags data for top/bottom query result + int16_t numOfCols; + struct SqlFunctionCtx **pCtx; +} SSubsidiaryResInfo; typedef struct SResultDataInfo { int16_t precision; @@ -187,7 +187,7 @@ typedef struct SqlFunctionCtx { void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SVariant tag; struct SResultRowEntryInfo *resultInfo; - SExtTagsInfo tagInfo; + SSubsidiaryResInfo subsidiaryRes; SPoint1 start; SPoint1 end; SFuncExecFuncs fpSet; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index ff1ddc88c0..f62ef30609 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -66,6 +66,7 @@ typedef enum ENodeType { QUERY_NODE_DATABLOCK_DESC, QUERY_NODE_SLOT_DESC, QUERY_NODE_COLUMN_DEF, + QUERY_NODE_DOWNSTREAM_SOURCE, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, @@ -98,6 +99,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_AGG, QUERY_NODE_LOGIC_PLAN_PROJECT, QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, + QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_SUBPLAN, QUERY_NODE_LOGIC_PLAN, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index d1bbbe7dbc..d8f8d55dc7 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -22,6 +22,7 @@ extern "C" { #include "querynodes.h" #include "query.h" +#include "tname.h" typedef struct SLogicNode { ENodeType type; @@ -67,12 +68,17 @@ typedef struct SProjectLogicNode { } SProjectLogicNode; typedef struct SVnodeModifLogicNode { - ENodeType type;; + SLogicNode node;; int32_t msgType; SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; } SVnodeModifLogicNode; +typedef struct SExchangeLogicNode { + SLogicNode node; + int32_t srcGroupId; +} SExchangeLogicNode; + typedef enum ESubplanType { SUBPLAN_TYPE_MERGE = 1, SUBPLAN_TYPE_PARTIAL, @@ -80,18 +86,28 @@ typedef enum ESubplanType { SUBPLAN_TYPE_MODIFY } ESubplanType; +typedef struct SSubplanId { + uint64_t queryId; + int32_t groupId; + int32_t subplanId; +} SSubplanId; + typedef struct SSubLogicPlan { ENodeType type; + SSubplanId id; SNodeList* pChildren; SNodeList* pParents; SLogicNode* pNode; ESubplanType subplanType; + SVgroupsInfo* pVgroupList; int32_t level; + int32_t splitFlag; } SSubLogicPlan; typedef struct SQueryLogicPlan { ENodeType type; - SNodeList* pSubplans; + int32_t totalLevel; + SNodeList* pTopSubplans; } SQueryLogicPlan; typedef struct SSlotDescNode { @@ -127,7 +143,7 @@ typedef struct SScanPhysiNode { int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC int32_t count; // repeat count int32_t reverse; // reverse scan count - SName tableName; + SName tableName; } SScanPhysiNode; typedef SScanPhysiNode SSystemTableScanPhysiNode; @@ -161,20 +177,21 @@ typedef struct SAggPhysiNode { SNodeList* pAggFuncs; } SAggPhysiNode; -typedef struct SDownstreamSource { +typedef struct SDownstreamSourceNode { + ENodeType type; SQueryNodeAddr addr; - uint64_t taskId; - uint64_t schedId; -} SDownstreamSource; + uint64_t taskId; + uint64_t schedId; +} SDownstreamSourceNode; typedef struct SExchangePhysiNode { - SPhysiNode node; - uint64_t srcTemplateId; // template id of datasource suplans - SArray* pSrcEndPoints; // SArray, scheduler fill by calling qSetSuplanExecutionNode + SPhysiNode node; + int32_t srcGroupId; // group id of datasource suplans + SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; typedef struct SDataSinkNode { - ENodeType type;; + ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; } SDataSinkNode; @@ -189,12 +206,6 @@ typedef struct SDataInserterNode { char *pData; } SDataInserterNode; -typedef struct SSubplanId { - uint64_t queryId; - int32_t templateId; - int32_t subplanId; -} SSubplanId; - typedef struct SSubplan { ENodeType type; SSubplanId id; // unique id of the subplan @@ -212,8 +223,8 @@ typedef struct SSubplan { typedef struct SQueryPlan { ENodeType type;; uint64_t queryId; - int32_t numOfSubplans; - SNodeList* pSubplans; // SNodeListNode. The execution level of subplan, starting from 0. + int32_t numOfSubplans; + SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. } SQueryPlan; #ifdef __cplusplus diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 0d62b7f0df..70e35824b6 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -31,14 +31,14 @@ typedef struct SPlanContext { int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList); // Set datasource of this subplan, multiple calls may be made to a subplan. -// @subplan subplan to be schedule -// @templateId templateId of a group of datasource subplans of this @subplan -// @ep one execution location of this group of datasource subplans -void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource); +// @pSubplan subplan to be schedule +// @groupId id of a group of datasource subplans of this @pSubplan +// @pSource one execution location of this group of datasource subplans +int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); // Convert to subplan to string for the scheduler to send to the executor -int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len); -int32_t qStringToSubplan(const char* str, SSubplan** subplan); +int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); +int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan); char* qQueryPlanToString(const SQueryPlan* pPlan); SQueryPlan* qStringToQueryPlan(const char* pStr); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index af6077208a..70e93efee1 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -43,10 +43,10 @@ enum { }; typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + int16_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema } STableComInfo; /* @@ -55,50 +55,46 @@ typedef struct STableComInfo { * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. */ typedef struct SCTableMeta { - int32_t vgId:24; + int32_t vgId : 24; int8_t tableType; uint64_t uid; uint64_t suid; } SCTableMeta; /* - * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. + * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a + * SCTableMeta. */ typedef struct STableMeta { - //BEGIN: KEEP THIS PART SAME WITH SCTableMeta - int32_t vgId:24; - int8_t tableType; - uint64_t uid; - uint64_t suid; - //END: KEEP THIS PART SAME WITH SCTableMeta - - // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info - int16_t sversion; - int16_t tversion; - STableComInfo tableInfo; - SSchema schema[]; + // BEGIN: KEEP THIS PART SAME WITH SCTableMeta + int32_t vgId : 24; + int8_t tableType; + uint64_t uid; + uint64_t suid; + // END: KEEP THIS PART SAME WITH SCTableMeta + + // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta + // info + int16_t sversion; + int16_t tversion; + STableComInfo tableInfo; + SSchema schema[]; } STableMeta; typedef struct SDBVgInfo { - int32_t vgVersion; + int32_t vgVersion; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT SHashObj *vgHash; //key:vgId, value:SVgroupInfo } SDBVgInfo; typedef struct SUseDbOutput { - char db[TSDB_DB_FNAME_LEN]; - uint64_t dbId; - SDBVgInfo *dbVgroup; + char db[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDBVgInfo* dbVgroup; } SUseDbOutput; -enum { - META_TYPE_NULL_TABLE = 1, - META_TYPE_CTABLE, - META_TYPE_TABLE, - META_TYPE_BOTH_TABLE -}; - +enum { META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE }; typedef struct STableMetaOutput { int32_t metaType; @@ -107,30 +103,30 @@ typedef struct STableMetaOutput { char ctbName[TSDB_TABLE_NAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN]; SCTableMeta ctbMeta; - STableMeta *tbMeta; + STableMeta* tbMeta; } STableMetaOutput; typedef struct SDataBuf { - void *pData; - uint32_t len; - void *handle; + void* pData; + uint32_t len; + void* handle; } SDataBuf; typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; //async callback function - void *param; - uint64_t requestId; - uint64_t requestObjRefId; - int32_t msgType; - SDataBuf msgInfo; + __async_send_cb_fn_t fp; // async callback function + void* param; + uint64_t requestId; + uint64_t requestObjRefId; + int32_t msgType; + SDataBuf msgInfo; } SMsgSendInfo; typedef struct SQueryNodeAddr { int32_t nodeId; // vgId or qnodeId - SEpSet epset; + SEpSet epSet; } SQueryNodeAddr; @@ -164,39 +160,72 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); * @param pInfo * @return */ -int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); -int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp); +int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp); void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); -bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); -int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta); +int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); +extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); -extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); -extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); - -#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE -#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE -#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE +#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE +#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE +#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE #define IS_CLIENT_RETRY_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH) #define IS_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT) - -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) +#define qFatal(...) \ + do { \ + if (qDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qError(...) \ + do { \ + if (qDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qWarn(...) \ + do { \ + if (qDebugFlag & DEBUG_WARN) { \ + taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qInfo(...) \ + do { \ + if (qDebugFlag & DEBUG_INFO) { \ + taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index fddf18c571..ccbeb00bfd 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -31,9 +31,9 @@ typedef int64_t SyncIndex; typedef uint64_t SyncTerm; typedef enum { - TAOS_SYNC_STATE_FOLLOWER = 0, - TAOS_SYNC_STATE_CANDIDATE = 1, - TAOS_SYNC_STATE_LEADER = 2, + TAOS_SYNC_STATE_FOLLOWER = 100, + TAOS_SYNC_STATE_CANDIDATE = 101, + TAOS_SYNC_STATE_LEADER = 102, } ESyncState; typedef struct SSyncBuffer { @@ -134,6 +134,7 @@ typedef struct SSyncInfo { SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; + char walPath[TSDB_FILENAME_LEN]; SSyncFSM* pFsm; void* rpcClient; diff --git a/include/os/osDir.h b/include/os/osDir.h index 223c603352..6cf28fb878 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -16,10 +16,24 @@ #ifndef _TD_OS_DIR_H_ #define _TD_OS_DIR_H_ +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define opendir OPENDIR_FUNC_TAOS_FORBID + #define readdir READDIR_FUNC_TAOS_FORBID + #define closedir CLOSEDIR_FUNC_TAOS_FORBID + #define dirname DIRNAME_FUNC_TAOS_FORBID + #undef basename + #define basename BASENAME_FUNC_TAOS_FORBID +#endif + #ifdef __cplusplus extern "C" { #endif +typedef struct TdDir *TdDirPtr; +typedef struct TdDirEntry *TdDirEntryPtr; + + void taosRemoveDir(const char *dirname); bool taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); @@ -27,6 +41,14 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); bool taosIsDir(const char *dirname); +char* taosDirName(char *dirname); +char* taosDirEntryBaseName(char *dirname); + +TdDirPtr taosOpenDir(const char *dirname); +TdDirEntryPtr taosReadDir(TdDirPtr pDir); +bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry); +char* taosGetDirEntryName(TdDirEntryPtr pDirEntry); +int32_t taosCloseDir(TdDirPtr pDir); #ifdef __cplusplus } diff --git a/include/os/osRand.h b/include/os/osRand.h index 422ea92a71..09e1f1b41d 100644 --- a/include/os/osRand.h +++ b/include/os/osRand.h @@ -20,7 +20,16 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define rand RAND_FUNC_TAOS_FORBID + #define srand SRAND_FUNC_TAOS_FORBID + #define rand_r RANDR_FUNC_TAOS_FORBID +#endif + +void taosSeedRand(uint32_t seed); uint32_t taosRand(void); +uint32_t taosRandR(uint32_t *pSeed); void taosRandStr(char* str, int32_t size); uint32_t taosSafeRand(void); diff --git a/include/os/osSocket.h b/include/os/osSocket.h index da1a9651e9..6544c89548 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -21,7 +21,10 @@ #define socket SOCKET_FUNC_TAOS_FORBID #define bind BIND_FUNC_TAOS_FORBID #define listen LISTEN_FUNC_TAOS_FORBID - // #define accept ACCEPT_FUNC_TAOS_FORBID + #define accept ACCEPT_FUNC_TAOS_FORBID + #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID + #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID + #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID #endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -38,31 +41,6 @@ extern "C" { #endif -#define TAOS_EPOLL_WAIT_TIME 500 -typedef int32_t SOCKET; -typedef SOCKET EpollFd; -#define EpollClose(pollFd) taosCloseSocket(pollFd) - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef SOCKET SocketFd; -#else -typedef int32_t SocketFd; -#endif - -int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); -int32_t taosWriteSocket(SocketFd fd, void *msg, int len); -int32_t taosReadSocket(SocketFd fd, void *msg, int len); -int32_t taosCloseSocketNoCheck(SocketFd fd); -int32_t taosCloseSocket(SocketFd fd); -void taosShutDownSocketRD(SOCKET fd); -void taosShutDownSocketWR(SOCKET fd); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); -int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); - -uint32_t taosInetAddr(const char *ipAddr); -const char *taosInetNtoa(struct in_addr ipInt); - #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #define htobe64 htonll #if defined(_TD_GO_DLL_) @@ -74,25 +52,54 @@ const char *taosInetNtoa(struct in_addr ipInt); #define htobe64 htonll #endif -int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); -int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); +#define TAOS_EPOLL_WAIT_TIME 500 -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(SOCKET sockFd); +typedef struct TdSocketServer *TdSocketServerPtr; +typedef struct TdSocket *TdSocketPtr; +typedef struct TdEpoll *TdEpollPtr; -void taosBlockSIGPIPE(); +int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); +int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); +int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosCloseSocket(TdSocketPtr *ppSocket); +int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); +int32_t taosShutDownSocketRD(TdSocketPtr pSocket); +int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer); +int32_t taosShutDownSocketWR(TdSocketPtr pSocket); +int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer); +int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket); +int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer); +int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on); +int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen); +int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); +int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); +int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); + +TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(TdSocketPtr pSocket); +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen); + +int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen); + +void taosBlockSIGPIPE(); uint32_t taosGetIpv4FromFqdn(const char *); int32_t taosGetFqdn(char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); -void taosIgnSIGPIPE(); -void taosSetMaskSIGPIPE(); +void taosIgnSIGPIPE(); +void taosSetMaskSIGPIPE(); +uint32_t taosInetAddr(const char *ipAddr); +const char *taosInetNtoa(struct in_addr ipInt); + +TdEpollPtr taosCreateEpoll(int32_t size); +int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event); +int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout); +int32_t taosCloseEpoll(TdEpollPtr *ppEpoll); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 1ebad370b5..54a3cfef7b 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -33,6 +33,7 @@ typedef struct { SDiskSize size; } SDiskSpace; +bool taosCheckSystemIsSmallEnd(); void taosGetSystemInfo(); int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 0f63a1f98c..9d6b559eee 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -208,7 +208,7 @@ void taos_init_imp(void) { atexit(taos_cleanup); errno = TSDB_CODE_SUCCESS; - srand(taosGetTimestampSec()); + taosSeedRand(taosGetTimestampSec()); deltaToUtcInitOnce(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 30ffaaee41..d4bcffbdc8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -173,6 +173,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg; pRequest->type = pMsgInfo->msgType; pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL}; + pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management STscObj* pTscObj = pRequest->pTscObj; SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); @@ -243,7 +244,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; - SQuery* pQuery; + SQuery* pQuery = NULL; SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; @@ -589,7 +590,7 @@ void* doFetchRow(SRequestObj* pRequest) { SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - epSet = pVgroupInfo->epset; + epSet = pVgroupInfo->epSet; } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { pRequest->type = TDMT_VND_SHOW_TABLES; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -606,7 +607,7 @@ void* doFetchRow(SRequestObj* pRequest) { pRequest->body.requestMsg.pData = pShowReq; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - epSet = pVgroupInfo->epset; + epSet = pVgroupInfo->epSet; int64_t transporterId = 0; STscObj* pTscObj = pRequest->pTscObj; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 78acbde5c8..04d80027ac 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE - #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -606,17 +604,17 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; - SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->consumeRsp; return pRsp->skipLogNum; } void tmqShowMsg(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - static bool noPrintSchema; - char pBuf[128]; - SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; - int32_t colNum = pRsp->schemas->nCols; + static bool noPrintSchema; + char pBuf[128]; + SMqPollRsp* pRsp = &tmq_message->consumeRsp; + int32_t colNum = pRsp->schemas->nCols; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { @@ -703,7 +701,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { goto WRITE_QUEUE_FAIL; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); + tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { /*printf("no data\n");*/ @@ -874,7 +872,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -886,7 +884,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien reqOffset = tmq->resetOffsetCfg; } - SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + SMqPollReq* pReq = malloc(sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; } @@ -900,7 +898,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + pReq->head.contLen = htonl(sizeof(SMqPollReq)); return pReq; } @@ -914,7 +912,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/ /*continue;*/ /*}*/ - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem @@ -941,7 +939,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqConsumeReq), + .len = sizeof(SMqPollReq), .handle = NULL, }; sendInfo->requestId = generateRequestId(); @@ -982,7 +980,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { if (vgStatus != TMQ_VG_STATUS__IDLE) { continue; } - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); @@ -1011,7 +1009,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqConsumeReq), + .len = sizeof(SMqPollReq), .handle = NULL, }; sendInfo->requestId = generateRequestId(); @@ -1271,7 +1269,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->consumeRsp; tDeleteSMqConsumeRsp(pRsp); /*free(tmq_message);*/ taosFreeQitem(tmq_message); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7a46261684..b0b0adbe78 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1568,7 +1568,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1; if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1; if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1; if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1; } @@ -1629,7 +1629,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1; if (tDecodeI32(pDecoder, &vgInfo.numOfTable) < 0) return -1; taosArrayPush(pRsp->pVgroupInfos, &vgInfo); } diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c index df705898ca..7ba272453c 100644 --- a/source/dnode/mgmt/daemon/src/dmnMain.c +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -97,6 +97,11 @@ int32_t dmnRunDnode() { } int main(int argc, char const *argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + uError("TDengine does not run on non-small-end machines."); + return -1; + } + if (dmnParseOption(argc, argv) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1b75d04b44..a17a45d46a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -924,10 +924,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { vgInfo.hashBegin = pVgroup->hashBegin; vgInfo.hashEnd = pVgroup->hashEnd; vgInfo.numOfTable = pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; - vgInfo.epset.numOfEps = pVgroup->replica; + vgInfo.epSet.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEp *pEp = &vgInfo.epset.eps[gid]; + SEp *pEp = &vgInfo.epSet.eps[gid]; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); @@ -935,7 +935,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { } mndReleaseDnode(pMnode, pDnode); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - vgInfo.epset.inUse = gid; + vgInfo.epSet.inUse = gid; } } vindex++; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 02eb5c2f24..e827810cc9 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -33,13 +33,17 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan); - SArray* pAray = NULL; - SArray* unassignedVg = pSub->unassignedVg; + if (pPlan == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } ASSERT(pSub->vgNum == 0); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; return -1; } @@ -47,6 +51,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t opNum = LIST_LENGTH(inner->pNodeList); if (opNum != 1) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; return -1; } SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); @@ -62,17 +68,24 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib pSub->vgNum++; plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup); + plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); SMqConsumerEp consumerEp = {0}; consumerEp.status = 0; consumerEp.consumerId = -1; - consumerEp.epSet = plan->execNode.epset; + consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; int32_t msgLen; - int32_t code = qSubPlanToString(plan, &consumerEp.qmsg, &msgLen); - taosArrayPush(unassignedVg, &consumerEp); + if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + taosArrayPush(pSub->unassignedVg, &consumerEp); } + qDestroyQueryPlan(pPlan); + return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b4359d0252..fa4fed59d0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -93,7 +93,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj strcpy(pSub->key, key); if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { - terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; tDeleteSMqSubscribeObj(pSub); free(pSub); return NULL; @@ -295,7 +294,11 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { for (int32_t k = 0; k < vgsz; k++) { char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + SMqSubVgEp vgEp = { + .epSet = pConsumerEp->epSet, + .vgId = pConsumerEp->vgId, + .offset = -1, + }; mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); if (pOffsetObj != NULL) { @@ -345,7 +348,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { - // TODO + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe)); @@ -412,7 +415,11 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { mInfo("mq rebalance will be triggered"); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg)}; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_MQ_DO_REBALANCE, + .pCont = pRebMsg, + .contLen = sizeof(SMqDoRebalanceMsg), + }; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } else { taosHashCleanup(pRebMsg->rebSubHash); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64c7a66bf9..d3642f4204 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -96,7 +96,11 @@ static void mndCalMqRebalance(void *param, void *tmrId) { if (mndIsMaster(pMnode)) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_MQ_TIMER, + .pCont = pReq, + .contLen = contLen, + }; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } @@ -631,4 +635,4 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 17fda48cd7..9dbc1be4e9 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -292,9 +292,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); - EXPECT_EQ(pInfo->epset.inUse, 0); - EXPECT_EQ(pInfo->epset.numOfEps, 1); - SEp* pAddr = &pInfo->epset.eps[0]; + EXPECT_EQ(pInfo->epSet.inUse, 0); + EXPECT_EQ(pInfo->epSet.numOfEps, 1); + SEp* pAddr = &pInfo->epSet.eps[0]; EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } @@ -307,9 +307,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); - EXPECT_EQ(pInfo->epset.inUse, 0); - EXPECT_EQ(pInfo->epset.numOfEps, 1); - SEp* pAddr = &pInfo->epset.eps[0]; + EXPECT_EQ(pInfo->epSet.inUse, 0); + EXPECT_EQ(pInfo->epSet.numOfEps, 1); + SEp* pAddr = &pInfo->epSet.eps[0]; EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 36626514ec..b4d45e83fb 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -13,16 +13,14 @@ * along with this program. If not, see . */ -#ifndef _TD_TQ_H_ -#define _TD_TQ_H_ +#ifndef _TQ_H_ +#define _TQ_H_ -#include "tcommon.h" #include "executor.h" -#include "tmallocator.h" #include "meta.h" -#include "scheduler.h" #include "taoserror.h" -#include "tlist.h" +#include "tcommon.h" +#include "tmallocator.h" #include "tmsg.h" #include "trpc.h" #include "ttimer.h" @@ -54,7 +52,7 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version); int tqCommit(STQ*); -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); @@ -62,4 +60,4 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg); } #endif -#endif /*_TD_TQ_H_*/ +#endif /*_TQ_H_*/ diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2d07a4e6ab..7189129d20 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -59,7 +59,7 @@ typedef struct { SWalCfg walCfg; uint32_t hashBegin; uint32_t hashEnd; - int8_t hashMethod; + int8_t hashMethod; } SVnodeCfg; typedef struct { @@ -205,6 +205,22 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); /* ------------------------- TQ READ --------------------------- */ +enum { + TQ_STREAM_TOKEN__DATA = 1, + TQ_STREAM_TOKEN__WATERMARK, + TQ_STREAM_TOKEN__CHECKPOINT, +}; + +typedef struct { + int8_t type; + int8_t reserved[7]; + union { + void *data; + int64_t wmTs; + int64_t checkpointId; + }; +} STqStreamToken; + STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) { diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index f416413859..fc6a3699d5 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -19,7 +19,7 @@ #include "meta.h" #include "tlog.h" #include "tq.h" -#include "trpc.h" +#include "tqPush.h" #ifdef __cplusplus extern "C" { @@ -31,30 +31,35 @@ extern "C" { taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ } \ } + #define tqError(...) \ { \ if (tqDebugFlag & DEBUG_ERROR) { \ taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ } \ } + #define tqWarn(...) \ { \ if (tqDebugFlag & DEBUG_WARN) { \ taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ } \ } + #define tqInfo(...) \ { \ if (tqDebugFlag & DEBUG_INFO) { \ taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \ } \ } + #define tqDebug(...) \ { \ if (tqDebugFlag & DEBUG_DEBUG) { \ taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \ } \ } + #define tqTrace(...) \ { \ if (tqDebugFlag & DEBUG_TRACE) { \ @@ -138,9 +143,7 @@ typedef struct { // topics that are not connectted STqMetaList* unconnectTopic; - // TODO:temporaral use, to be replaced by unified tfile TdFilePtr pFile; - // TODO:temporaral use, to be replaced by unified tfile TdFilePtr pIdxFile; char* dirPath; @@ -157,6 +160,7 @@ struct STQ { STqCfg* tqConfig; STqMemRef tqMemRef; STqMetaStore* tqMeta; + STqPushMgr* tqPushMgr; SWal* pWal; SMeta* pVnodeMeta; }; diff --git a/source/dnode/vnode/src/inc/tqPush.h b/source/dnode/vnode/src/inc/tqPush.h new file mode 100644 index 0000000000..a6121c5dc1 --- /dev/null +++ b/source/dnode/vnode/src/inc/tqPush.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TQ_PUSH_H_ +#define _TQ_PUSH_H_ + +#include "executor.h" +#include "thash.h" +#include "trpc.h" +#include "ttimer.h" +#include "vnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + TQ_PUSHER_TYPE__CLIENT = 1, + TQ_PUSHER_TYPE__STREAM, +}; + +typedef struct { + int8_t type; + int8_t reserved[3]; + int32_t ttl; + int64_t consumerId; + SRpcMsg* pMsg; + // SMqPollRsp* rsp; +} STqClientPusher; + +typedef struct { + int8_t type; + int8_t nodeType; + int8_t reserved[6]; + int64_t streamId; + qTaskInfo_t task; + // TODO sync function +} STqStreamPusher; + +typedef struct { + int8_t type; // mq or stream +} STqPusher; + +typedef struct { + SHashObj* pHash; // +} STqPushMgr; + +typedef struct { + int8_t inited; + tmr_h timer; +} STqPushMgmt; + +static STqPushMgmt tqPushMgmt; + +int32_t tqPushMgrInit(); +void tqPushMgrCleanUp(); + +STqPushMgr* tqPushMgrOpen(); +void tqPushMgrClose(STqPushMgr* pushMgr); + +STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl); +STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet); + +#ifdef __cplusplus +} +#endif + +#endif /*_TQ_PUSH_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 96a76ea7d4..1451ac9685 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -52,6 +52,7 @@ struct STsdb { STsdbFS * fs; SMeta * pMeta; STfs * pTfs; + SSmaStat * pSmaStat; }; #define REPO_ID(r) ((r)->vgId) diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 641255a294..173e991631 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -42,7 +42,10 @@ typedef struct { typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array - SArray * smaf; // sma data file array + + // SArray * v2f100.tsma.index_name + + SArray * smaf; // sma data file array v2f1900.tsma.index_name } SFSStatus; typedef struct { diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index e4de7a6685..6e4ad909ae 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,6 +16,8 @@ #ifndef _TD_TSDB_SMA_H_ #define _TD_TSDB_SMA_H_ +typedef struct SSmaStat SSmaStat; + // insert/update interface int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData); int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData); @@ -26,13 +28,14 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData); int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult); // management interface -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void* result); +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg); +int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); - - - +int32_t tsdbFreeSmaState(SSmaStat *pSmaStat); // internal func + + static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) { int32_t len = 0; len += taosEncodeFixedU64(pData, tableUid); diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index f49515412b..efdb3e0fe4 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -923,6 +923,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { SMetaDB *pDB = pMeta->pDB; DBC * pCur = NULL; DBT pkey = {0}, pval = {0}; + uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP; int ret; pUids = taosArrayInit(16, sizeof(tb_uid_t)); @@ -941,13 +942,8 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { void *pBuf = NULL; // TODO: lock? - while (true) { - ret = pCur->get(pCur, &pkey, &pval, isDup ? DB_NEXT_DUP : DB_NEXT_NODUP); - if(ret == 0) { + while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) { taosArrayPush(pUids, pkey.data); - continue; - } - break; } if (pCur) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d540b589b7..d15481b4aa 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -12,28 +12,16 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -int tqInit() { - int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); - if (old == 1) return 0; +int32_t tqInit() { return tqPushMgrInit(); } - tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); - return 0; -} +void tqCleanUp() { tqPushMgrCleanUp(); } -void tqCleanUp() { - int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0); - if (old == 0) return; - taosTmrStop(tqMgmt.timer); - taosTmrCleanUp(tqMgmt.timer); -} - -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -42,7 +30,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->pWal = pWal; - pTq->pVnodeMeta = pMeta; + pTq->pVnodeMeta = pVnodeMeta; #if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); @@ -60,6 +48,13 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl return NULL; } + pTq->tqPushMgr = tqPushMgrOpen(); + if (pTq->tqPushMgr == NULL) { + // free store + free(pTq); + return NULL; + } + return pTq; } @@ -72,6 +67,28 @@ void tqClose(STQ* pTq) { } int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { + if (msgType != TDMT_VND_SUBMIT) return 0; + void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL); + while (pIter != NULL) { + STqPusher* pusher = *(STqPusher**)pIter; + if (pusher->type == TQ_PUSHER_TYPE__STREAM) { + STqStreamPusher* streamPusher = (STqStreamPusher*)pusher; + // repack + STqStreamToken* token = malloc(sizeof(STqStreamToken)); + if (token == NULL) { + taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + token->type = TQ_STREAM_TOKEN__DATA; + token->data = msg; + // set input + // exec + } + // send msg to ep + } + // iterate hash + // process all msg // if waiting // memcpy and send msg to fetch thread // TODO: add reference @@ -199,7 +216,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu for (int j = 0; j < TQ_BUFFER_SIZE; j++) { pTopic->buffer.output[j].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); - SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta}; + SReadHandle handle = { + .reader = pReadHandle, + .meta = pTq->pVnodeMeta, + }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); } @@ -208,11 +228,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { - SMqConsumeReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t fetchOffset; - int64_t blockingTime = pReq->blockingTime; +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { + SMqPollReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t fetchOffset; + int64_t blockingTime = pReq->blockingTime; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = 0; @@ -222,7 +242,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { fetchOffset = pReq->currentOffset + 1; } - SMqConsumeRsp rsp = { + SMqPollRsp rsp = { .consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL, @@ -236,6 +256,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { rpcSendResponse(pMsg); return 0; } + int sz = taosArrayGetSize(pConsumer->topics); ASSERT(sz == 1); STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0); @@ -247,13 +268,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SWalHead* pHead; while (1) { - int8_t pos = fetchOffset % TQ_BUFFER_SIZE; + /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user break; } + int8_t pos = fetchOffset % TQ_BUFFER_SIZE; pHead = pTopic->pReadhandle->pHead; if (pHead->head.msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; @@ -280,7 +302,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { rsp.numOfTopics = 1; rsp.pBlockData = pRes; - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; @@ -290,7 +312,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ((SMqRspHead*)buf)->epoch = pReq->epoch; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqConsumeRsp(&abuf, &rsp); + tEncodeSMqPollRsp(&abuf, &rsp); taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); pMsg->pCont = buf; pMsg->contLen = tlen; @@ -304,7 +326,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } } - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp); + /*if (blockingTime != 0) {*/ + /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/ + /*} else {*/ + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; @@ -314,12 +339,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ((SMqRspHead*)buf)->epoch = pReq->epoch; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqConsumeRsp(&abuf, &rsp); + tEncodeSMqPollRsp(&abuf, &rsp); rsp.pBlockData = NULL; pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; rpcSendResponse(pMsg); + /*}*/ + return 0; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c new file mode 100644 index 0000000000..4186f29e2a --- /dev/null +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tqPush.h" + +int32_t tqPushMgrInit() { + // + int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1); + if (old == 1) return 0; + + tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); + return 0; +} + +void tqPushMgrCleanUp() { + int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0); + if (old == 0) return; + taosTmrStop(tqPushMgmt.timer); + taosTmrCleanUp(tqPushMgmt.timer); +} + +STqPushMgr* tqPushMgrOpen() { + STqPushMgr* mgr = malloc(sizeof(STqPushMgr)); + if (mgr == NULL) { + return NULL; + } + mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return mgr; +} + +void tqPushMgrClose(STqPushMgr* pushMgr) { + taosHashCleanup(pushMgr->pHash); + free(pushMgr); +} + +STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) { + STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher)); + if (clientPusher == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + clientPusher->type = TQ_PUSHER_TYPE__CLIENT; + clientPusher->pMsg = pMsg; + clientPusher->consumerId = consumerId; + clientPusher->ttl = ttl; + if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(clientPusher); + // TODO send rsp back + return NULL; + } + return clientPusher; +} + +STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) { + STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher)); + if (streamPusher == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + streamPusher->type = TQ_PUSHER_TYPE__STREAM; + streamPusher->nodeType = 0; + streamPusher->streamId = streamId; + /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/ + + if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(streamPusher); + return NULL; + } + return streamPusher; +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2f24df0309..92a111298f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "vnode.h" @@ -37,6 +36,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t pMsg->length = htonl(pMsg->length); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + // iterate and convert if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; while (true) { if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; diff --git a/source/libs/parser/inc/insertParser.h b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c similarity index 70% rename from source/libs/parser/inc/insertParser.h rename to source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index 311db19bcd..f2f48bbc8a 100644 --- a/source/libs/parser/inc/insertParser.h +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -12,20 +12,3 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - -#ifndef TDENGINE_INSERTPARSER_H -#define TDENGINE_INSERTPARSER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "parser.h" - -int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_INSERTPARSER_H diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 74fb8c1c1f..00e97c7b61 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -365,7 +365,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); - if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) { + if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { tfree(s); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 2d8c470113..1b3e00f090 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -89,6 +89,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, static void tsdbFree(STsdb *pTsdb) { if (pTsdb) { tsdbFreeFS(pTsdb->fs); + tsdbFreeSmaState(pTsdb->pSmaStat); tfree(pTsdb->path); free(pTsdb); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index b465dc3a88..c5f1261282 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -21,6 +21,10 @@ #define SMA_STORE_SINGLE_BLOCKS // store SMA data by single block or multiple blocks +#define SMA_STATE_HASH_SLOT 4 +#define SMA_STATE_ITEM_HASH_SLOT 32 + +#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test typedef enum { SMA_STORAGE_LEVEL_TSDB = 0, // store TSma in dir e.g. vnode${N}/tsdb/.tsma SMA_STORAGE_LEVEL_DFILESET = 1 // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name} @@ -48,6 +52,22 @@ typedef struct { // TODO } STSmaReadH; +typedef struct { + /** + * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from + * Streaming Module or TSDB local persistence. + */ + int8_t state; // ETsdbSmaStat + SHashObj *expiredWindows; // key: skey of time window, value: N/A +} SSmaStatItem; + +struct SSmaStat { + SHashObj *smaStatItems; // key: indexName, value: SSmaStatItem +}; + // declaration of static functions static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); @@ -64,6 +84,125 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin); +static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { + ASSERT(pSmaStat != NULL); + + if (*pSmaStat != NULL) { // no lock + return TSDB_CODE_SUCCESS; + } + + // TODO: lock. lazy mode when update expired window, or hungry mode during tsdbNew. + if (*pSmaStat == NULL) { + *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); + if (*pSmaStat == NULL) { + // TODO: unlock + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + (*pSmaStat)->smaStatItems = + taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + if ((*pSmaStat)->smaStatItems == NULL) { + tfree(*pSmaStat); + // TODO: unlock + return TSDB_CODE_FAILED; + } + } + // TODO: unlock + return TSDB_CODE_SUCCESS; +} + +static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { + SSmaStatItem *pItem = NULL; + + pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem)); + if (pItem) { + pItem->state = state; + pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), + true, HASH_ENTRY_LOCK); + if (!pItem->expiredWindows) { + tfree(pItem); + } + } + return pItem; +} + +int32_t tsdbFreeSmaState(SSmaStat *pSmaStat) { + if (pSmaStat) { + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL); + while (item != NULL) { + taosHashCleanup(item->expiredWindows); + item = taosHashIterate(pSmaStat->smaStatItems, item); + } + + taosHashCleanup(pSmaStat->smaStatItems); + free(pSmaStat); + } +} + +/** + * @brief Update expired window according to msg from stream computing module. + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { + if (msg == NULL) { + return TSDB_CODE_FAILED; + } + + tsdbInitSmaStat(&pTsdb->pSmaStat); // lazy mode + + // TODO: decode the msg => start + const char * indexName = SMA_TEST_INDEX_NAME; + const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; + TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; + int64_t now = taosGetTimestampMs(); + for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { + expiredWindows[i] = now + i; + } + + // TODO: decode the msg <= end + SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems; + + SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName)); + if (!pItem) { + pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state + if (!pItem) { + // Response to stream computing: OOM + // For query, if the indexName not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + if (taosHashPut(pItemsHash, indexName, strnlen(indexName, TSDB_INDEX_NAME_LEN), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + free(pItem); + return TSDB_CODE_FAILED; + } + } + + int8_t state = TSDB_SMA_STAT_EXPIRED; + for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { + if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosHashRemove(pItemsHash, indexName, sizeof(indexName)); + return TSDB_CODE_FAILED; + } + } + + return TSDB_CODE_SUCCESS; +} + /** * @brief Judge the tSma storage level * @@ -484,6 +623,22 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow * @return int32_t */ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) { + const char *indexName = param->indexName; + + SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName)); + if (pItem == NULL) { + // mark all window as expired and notify query module to query raw TS data. + return TSDB_CODE_SUCCESS; + } + + int32_t nQueryWin = 0; + for (int32_t n = 0; n < nQueryWin; ++n) { + TSKEY thisWindow = n; + if (taosHashGet(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)) != NULL) { + // TODO: mark this window as expired. + } + } + STSmaReadH tReadH = {0}; tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ac10f78a8a..92b56a54a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -33,7 +33,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; switch (pMsg->msgType) { - case TDMT_VND_QUERY:{ + case TDMT_VND_QUERY: { return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); } case TDMT_VND_QUERY_CONTINUE: @@ -67,7 +67,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: - return tqProcessConsumeReq(pVnode->pTq, pMsg); + return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: @@ -77,8 +77,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { - STbCfg * pTbCfg = NULL; - STbCfg * pStbCfg = NULL; + STbCfg *pTbCfg = NULL; + STbCfg *pStbCfg = NULL; tb_uid_t uid; int32_t nCols; int32_t nTagCols; @@ -210,9 +210,9 @@ static void freeItemHelper(void *pItem) { */ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { SMTbCursor *pCur = metaOpenTbCursor(pVnode->pMeta); - SArray * pArray = taosArrayInit(10, POINTER_BYTES); + SArray *pArray = taosArrayInit(10, POINTER_BYTES); - char * name = NULL; + char *name = NULL; int32_t totalLen = 0; int32_t numOfTables = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index d3c9b50e4a..4f15185254 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -168,10 +168,10 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { } TEST_F(TqMetaUpdateAppendTest, multiplePage) { - srand(0); + taosSeedRand(0); std::vector v; for (int i = 0; i < 1000; i++) { - v.push_back(rand()); + v.push_back(taosRand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -202,10 +202,10 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) { } TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { - srand(0); + taosSeedRand(0); std::vector v; for (int i = 0; i < 1000; i++) { - v.push_back(rand()); + v.push_back(taosRand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -213,14 +213,14 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); - v[i] = rand(); + v[i] = taosRand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } for (int i = 500; i < 1000; i++) { - v[i] = rand(); + v[i] = taosRand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -235,7 +235,7 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { ASSERT(pMeta); for (int i = 500; i < 1000; i++) { - v[i] = rand(); + v[i] = taosRand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -250,10 +250,10 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { } TEST_F(TqMetaUpdateAppendTest, dupCommit) { - srand(0); + taosSeedRand(0); std::vector v; for (int i = 0; i < 1000; i++) { - v.push_back(rand()); + v.push_back(taosRand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index fb378928b6..81e68e5433 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -929,7 +929,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMg }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index aeac6b3eee..00f6a508b7 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -228,10 +228,10 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { vgInfo.vgId = i + 1; vgInfo.hashBegin = i * hashUnit; vgInfo.hashEnd = hashUnit * (i + 1) - 1; - vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vgInfo.epset.inUse = i % vgInfo.epset.numOfEps; - for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) { - SEp *addr = &vgInfo.epset.eps[n]; + vgInfo.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1; + vgInfo.epSet.inUse = i % vgInfo.epSet.numOfEps; + for (int32_t n = 0; n < vgInfo.epSet.numOfEps; ++n) { + SEp *addr = &vgInfo.epSet.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = n + 22; } @@ -301,10 +301,10 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * vg.hashEnd = htonl(UINT32_MAX); } - vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vg.epset.inUse = i % vg.epset.numOfEps; - for (int32_t n = 0; n < vg.epset.numOfEps; ++n) { - SEp *addr = &vg.epset.eps[n]; + vg.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg.epSet.inUse = i % vg.epSet.numOfEps; + for (int32_t n = 0; n < vg.epSet.numOfEps; ++n) { + SEp *addr = &vg.epSet.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = n + 22; } @@ -723,7 +723,7 @@ void *ctgTestGetDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); @@ -747,7 +747,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -771,7 +771,7 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -801,7 +801,7 @@ void *ctgTestGetCtableMetaThread(void *param) { tfree(tbMeta); if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { @@ -838,7 +838,7 @@ void *ctgTestSetCtableMetaThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -877,7 +877,7 @@ TEST(tableMeta, normalTable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { usleep(50000); @@ -1384,7 +1384,7 @@ TEST(refreshGetMeta, normal2normal) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1463,7 +1463,7 @@ TEST(refreshGetMeta, normal2notexist) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1537,7 +1537,7 @@ TEST(refreshGetMeta, normal2child) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1621,7 +1621,7 @@ TEST(refreshGetMeta, stable2child) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1706,7 +1706,7 @@ TEST(refreshGetMeta, stable2stable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1794,7 +1794,7 @@ TEST(refreshGetMeta, child2stable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1879,7 +1879,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 8); - ASSERT_EQ(vgInfo->epset.numOfEps, 3); + ASSERT_EQ(vgInfo->epSet.numOfEps, 3); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1921,7 +1921,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 9); - ASSERT_EQ(vgInfo->epset.numOfEps, 4); + ASSERT_EQ(vgInfo->epSet.numOfEps, 4); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1964,13 +1964,13 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 1); - ASSERT_EQ(vgInfo->epset.numOfEps, 1); + ASSERT_EQ(vgInfo->epSet.numOfEps, 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1); ASSERT_EQ(vgInfo->vgId, 2); - ASSERT_EQ(vgInfo->epset.numOfEps, 2); + ASSERT_EQ(vgInfo->epSet.numOfEps, 2); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2); ASSERT_EQ(vgInfo->vgId, 3); - ASSERT_EQ(vgInfo->epset.numOfEps, 3); + ASSERT_EQ(vgInfo->epSet.numOfEps, 3); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2025,14 +2025,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->epset.numOfEps, 3); + ASSERT_EQ(pvgInfo->epSet.numOfEps, 3); taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); @@ -2053,14 +2053,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); - ASSERT_EQ(vgInfo.epset.numOfEps, 2); + ASSERT_EQ(vgInfo.epSet.numOfEps, 2); code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->epset.numOfEps, 3); + ASSERT_EQ(pvgInfo->epSet.numOfEps, 3); taosArrayDestroy(vgList); catalogDestroy(); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f35cfaee70..b34067ba4e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -92,10 +92,11 @@ typedef struct SResultRowPool { struct STaskAttr; struct STaskRuntimeEnv; struct SUdfInfo; +struct SqlFunctionCtx; int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); -size_t getResultRowSize(SArray* pExprInfo); +size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 43e6adf591..56f222c58a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -441,17 +441,24 @@ typedef struct SStreamBlockScanInfo { } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { - void *pTransporter; - SEpSet epSet; - int32_t type; // show type - tsem_t ready; - void *readHandle; - SSchema *pSchema; - SSDataBlock *pRes; - int64_t numOfBlocks; // extract basic running information. - int64_t totalRows; - int64_t elapsedTime; - int64_t totalBytes; + union { + void* pTransporter; + void* readHandle; + }; + + void *pCur; // cursor + SRetrieveTableReq* pReq; + SEpSet epSet; + int32_t type; // show type + tsem_t ready; + SSchema* pSchema; + SSDataBlock* pRes; + + int32_t capacity; + int64_t numOfBlocks; // extract basic running information. + int64_t totalRows; + int64_t elapsedTime; + int64_t totalBytes; } SSysTableScanInfo; typedef struct SOptrBasicInfo { @@ -462,12 +469,14 @@ typedef struct SOptrBasicInfo { int32_t capacity; } SOptrBasicInfo; +//TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; typedef struct STableIntervalOperatorInfo { @@ -623,14 +632,15 @@ typedef struct SOrderOperatorInfo { uint64_t totalElapsed; // total elapsed time } SOrderOperatorInfo; -SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSystemScanOperatorInfo(void* pSystemTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, + int32_t tableType, SEpSet epset, SExecTaskInfo* pTaskInfo); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1b901ee9f6..9d77e23d38 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -46,7 +46,7 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t size = 0; for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - size += pQueryAttr->pExpr1[i].base.interBytes; +// size += pQueryAttr->pExpr1[i].base.interBytes; } assert(size >= 0); @@ -172,9 +172,14 @@ SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_ return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]); } -size_t getResultRowSize(SArray* pExprInfo) { - size_t numOfOutput = taosArrayGetSize(pExprInfo); - return (numOfOutput * sizeof(SResultRowEntryInfo)) + /*pQueryAttr->interBufSize +*/ sizeof(SResultRow); +size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { + int32_t rowSize = (numOfOutput * sizeof(SResultRowEntryInfo)) + sizeof(SResultRow); + + for(int32_t i = 0; i < numOfOutput; ++i) { + rowSize += pCtx[i].resDataInfo.interBufSize; + } + + return rowSize; } SResultRowPool* initResultRowPool(size_t size) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 480114480c..326f7ca3e9 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -64,7 +64,7 @@ typedef enum SResultTsInterpType { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { - uint32_t v = rand(); + uint32_t v = taosRand(); if (v % 1000 <= 0) { return NULL; @@ -74,7 +74,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { } static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { - uint32_t v = rand(); + uint32_t v = taosRand(); if (v % 1000 <= 0) { return NULL; } else { @@ -83,7 +83,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { } static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { - uint32_t v = rand(); + uint32_t v = taosRand(); if (v % 5 <= 1) { return NULL; } else { @@ -212,6 +212,7 @@ static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); +static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput); static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; @@ -346,6 +347,7 @@ SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.blockId = pNode->dataBlockId; + pBlock->info.rowSize = pNode->resultRowSize; for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; @@ -682,12 +684,6 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t w.ekey = w.skey + pInterval->interval - 1; } } - - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ -// ASSERT(win->skey <= win->ekey); // todo no need this return w; } @@ -1920,9 +1916,9 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { } } if (p != NULL) { - p->tagInfo.pTagCtxList = pTagCtx; - p->tagInfo.numOfTagCols = num; - p->tagInfo.tagsLen = tagLen; + p->subsidiaryRes.pCtx = pTagCtx; + p->subsidiaryRes.numOfCols = num; + p->subsidiaryRes.bufLen = tagLen; } else { tfree(pTagCtx); } @@ -1969,7 +1965,7 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI pCtx->order = pQueryAttr->order.order; // pCtx->functionId = pFunct->functionId; pCtx->stableQuery = pQueryAttr->stableQuery; - pCtx->resDataInfo.interBufSize = pFunct->interBytes; +// pCtx->resDataInfo.interBufSize = pFunct->interBytes; pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; @@ -2051,7 +2047,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC SExprBasicInfo *pFunct = &pExpr->base; SqlFunctionCtx* pCtx = &pFuncCtx[i]; - fmGetFuncExecFuncs(pExpr->pExpr->_function.functionId, &pCtx->fpSet); + fmGetFuncExecFuncs(pExpr->pExpr->_function.pFunctNode->funcId, &pCtx->fpSet); pCtx->input.numOfInputCols = pFunct->numOfParams; pCtx->input.pData = calloc(pFunct->numOfParams, POINTER_BYTES); @@ -2061,8 +2057,6 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; pCtx->resDataInfo.type = pFunct->resSchema.type; pCtx->order = TSDB_ORDER_ASC; -// pCtx->functionId = pExpr->pExpr->_function.pFunctNode->;//TODO remove it - pCtx->stableQuery = false; // TODO pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; @@ -2119,15 +2113,14 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC } for(int32_t i = 1; i < numOfOutput; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i - 1); - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr->base.interBytes); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i].resDataInfo.interBufSize); } setCtxTagColumnInfo(pFuncCtx, numOfOutput); return pFuncCtx; } -static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -2138,7 +2131,7 @@ static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].tagInfo.pTagCtxList); + tfree(pCtx[i].subsidiaryRes.pCtx); } tfree(pCtx); @@ -2222,46 +2215,6 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { } } -static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; - - //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - - //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); -// destroyUdfInfo(pRuntimeEnv->pUdfInfo); - destroyDiskbasedBuf(pRuntimeEnv->pResultBuf); - doFreeQueryHandle(pRuntimeEnv); - - destroyTsComp(pRuntimeEnv, pQueryAttr); - - pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); - - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); - - taosHashCleanup(pRuntimeEnv->pResultRowHashTable); - pRuntimeEnv->pResultRowHashTable = NULL; - - taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); - pRuntimeEnv->pTableRetrieveTsMap = NULL; - - taosHashCleanup(pRuntimeEnv->pResultRowListSet); - pRuntimeEnv->pResultRowListSet = NULL; - - destroyOperatorInfo(pRuntimeEnv->proot); - - pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); - taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); - taosArrayDestroy(pRuntimeEnv->pResultRowArrayList); - pRuntimeEnv->prevResult = NULL; -} - -static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { - return pQInfo->rspContext != NULL; -} - bool isTaskKilled(SExecTaskInfo *pTaskInfo) { // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. @@ -3386,22 +3339,17 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. - */ struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); cleanupResultRowEntry(pEntry); pCtx[i].resultInfo = pEntry; - pCtx[i].pOutput = pData->pData; pCtx[i].currentStage = stage; // set the timestamp output buffer for top/bottom/diff query - int32_t fid = pCtx[i].functionId; - if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } +// int32_t fid = pCtx[i].functionId; +// if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { +// if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; +// } } initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); @@ -3709,7 +3657,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.tupleSize); + int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.rowSize); if (ret != TSDB_CODE_SUCCESS) { return; } @@ -4972,11 +4920,11 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf return pTaskInfo->code; } - SDownstreamSource *pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SDownstreamSourceNode *pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); SSourceDataInfo *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); @@ -4999,7 +4947,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo); + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); return TSDB_CODE_SUCCESS; } @@ -5070,7 +5018,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SEx } SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, i); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); SSDataBlock* pRes = pExchangeInfo->pResult; @@ -5167,7 +5115,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) { tsem_wait(&pExchangeInfo->ready); SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); SRetrieveTableRsp* pRsp = pDataInfo->pRsp; if (pRsp->numOfRows == 0) { @@ -5233,9 +5181,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { #endif } -static SSDataBlock* createResultDataBlock(const SArray* pExprInfo); - -SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { +// TODO handle the error +SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -5246,9 +5193,20 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* return NULL; } - size_t numOfSources = taosArrayGetSize(pSources); + size_t numOfSources = LIST_LENGTH(pSources); + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + if (pInfo->pSources == NULL) { + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfSources; ++i) { + SNodeListNode* pNode = nodesListGetNode((SNodeList*) pSources, i); + taosArrayPush(pInfo->pSources, pNode); + } - pInfo->pSources = taosArrayDup(pSources); pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { tfree(pInfo); @@ -5268,8 +5226,8 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); } - size_t size = taosArrayGetSize(pExprInfo); - pInfo->pResult = createResultDataBlock(pExprInfo); + size_t size = pBlock->info.numOfCols; + pInfo->pResult = pBlock; pInfo->seqLoadData = true; tsem_init(&pInfo->ready, 0, 0); @@ -5323,11 +5281,12 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { SColumnInfoData colInfoData = {0}; SExprInfo* p = taosArrayGetP(pExprInfo, i); - SSchema* pSchema = &p->base.resSchema; + SResSchema* pSchema = &p->base.resSchema; colInfoData.info.type = pSchema->type; colInfoData.info.colId = pSchema->colId; colInfoData.info.bytes = pSchema->bytes; - + colInfoData.info.scale = pSchema->scale; + colInfoData.info.precision = pSchema->precision; taosArrayPush(pResult, &colInfoData); } @@ -5475,38 +5434,67 @@ static SSDataBlock* doSysTableScan(void* param, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; - SRetrieveTableReq* req = calloc(1, sizeof(SRetrieveTableReq)); - if (req == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + // retrieve local table list info from vnode + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); + } + + SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, 0); + + char * name = NULL; + int32_t numOfRows = 0; + while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { + colDataAppend(pTableNameCol, numOfRows, name, false); + numOfRows += 1; + if (numOfRows >= pInfo->capacity) { + break; + } + } + + pInfo->totalRows += numOfRows; + pInfo->pRes->info.rows = numOfRows; + +// pInfo->elapsedTime; +// pInfo->totalBytes; + return (pInfo->pRes->info.rows == 0)? NULL:pInfo->pRes; + } else { // load the meta from mnode of the given epset + if (pInfo->pReq == NULL) { + pInfo->pReq = calloc(1, sizeof(SRetrieveTableReq)); + if (pInfo->pReq == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pInfo->pReq->type = pInfo->type; + } + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pMsgSendInfo->param = NULL; + pMsgSendInfo->msgInfo.pData = pInfo->pReq; + pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); + pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); + + tsem_wait(&pInfo->ready); + // handle the response and return to the caller } - req->type = pInfo->type; - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pMsgSendInfo->param = NULL; - pMsgSendInfo->msgInfo.pData = req; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); - pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); - - tsem_wait(&pInfo->ready); - // handle the response and return to the caller - return NULL; } -SOperatorInfo* createSystemScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, + int32_t tableType, SEpSet epset, SExecTaskInfo* pTaskInfo) { SSysTableScanInfo* pInfo = calloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5516,6 +5504,17 @@ SOperatorInfo* createSystemScanOperatorInfo(void* pSysTableReadHandle, const SAr return NULL; } + // todo: create the schema of result data block + pInfo->capacity = 4096; + pInfo->type = tableType; + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + pInfo->readHandle = pSysTableReadHandle; + blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); + } else { + tsem_init(&pInfo->ready, 0, 0); + pInfo->epSet = epset; + } + pInfo->readHandle = pSysTableReadHandle; pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; @@ -5524,6 +5523,7 @@ SOperatorInfo* createSystemScanOperatorInfo(void* pSysTableReadHandle, const SAr pOperator->info = pInfo; pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->nextDataFn = doSysTableScan; + pOperator->closeFn = destroySysTableScannerOperatorInfo; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -5661,7 +5661,7 @@ SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { return pOrderColumns; } -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SArray* pExprInfo); +static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput); static void clearupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { @@ -6042,7 +6042,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t goto _error; } - int32_t code = doInitAggInfoSup(&pInfo->aggSup, pExprInfo); + int32_t code = doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfOutput); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -7083,13 +7083,14 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SArray* pExprInfo) { +int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - pAggSup->pool = initResultRowPool(getResultRowSize(pExprInfo)); + pAggSup->pool = initResultRowPool(pAggSup->resultRowSize); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || @@ -7113,7 +7114,7 @@ static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t n pInfo->binfo.pRes = pResultBlock; pInfo->binfo.capacity = numOfRows; - doInitAggInfoSup(&pInfo->aggSup, pExprInfo); + doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, taosArrayGetSize(pExprInfo)); pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); int32_t index = 0; @@ -7165,7 +7166,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { assert(pInfo != NULL); - destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); + destroySqlFunctionCtx(pInfo->pCtx, numOfOutput); tfree(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); @@ -7185,6 +7186,7 @@ static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } + static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); @@ -7233,6 +7235,16 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } +static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput) { + SSysTableScanInfo* pInfo = (SSysTableScanInfo*) param; + tsem_destroy(&pInfo->ready); + blockDataDestroy(pInfo->pRes); + + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + metaCloseTbCursor(pInfo->pCur); + } +} + SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); @@ -7340,14 +7352,15 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - doInitAggInfoSup(&pInfo->aggSup, pExprInfo); + size_t numOfOutput = taosArrayGetSize(pExprInfo); + doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfOutput); - pInfo->order = TSDB_ORDER_ASC; + pInfo->order = TSDB_ORDER_ASC; pInfo->precision = TSDB_TIME_PRECISION_MICRO; - pInfo->win = pTaskInfo->window; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->interval = *pInterval; - int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); + int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); @@ -8008,9 +8021,19 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -SArray* createExprInfo(SAggPhysiNode* pPhyNode, int32_t* resultRowSize) { - *resultRowSize = pPhyNode->node.pOutputDataBlockDesc->resultRowSize; +static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision, const char* name) { + SResSchema s = {0}; + s.scale = scale; + s.precision = precision; + s.type = type; + s.bytes = bytes; + s.colId = slotId; + strncpy(s.name, name, tListLen(s.name)); + return s; +} + +SArray* createExprInfo(SAggPhysiNode* pPhyNode) { int32_t numOfAggFuncs = LIST_LENGTH(pPhyNode->pAggFuncs); SArray* pArray = taosArrayInit(numOfAggFuncs, POINTER_BYTES); @@ -8026,13 +8049,16 @@ SArray* createExprInfo(SAggPhysiNode* pPhyNode, int32_t* resultRowSize) { pExp->base.pParam[0].pCol = calloc(1, sizeof(SColumn)); SColumn* pCol = pExp->base.pParam[0].pCol; - ASSERT(LIST_LENGTH(pPhyNode->pAggFuncs) == 1); - STargetNode* pTargetNode = (STargetNode*) nodesListGetNode(pPhyNode->pAggFuncs, 0); + STargetNode* pTargetNode = (STargetNode*) nodesListGetNode(pPhyNode->pAggFuncs, i); + ASSERT(pTargetNode->slotId == i); SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; - pExp->base.resSchema = createSchema(pFuncNode->node.resType.type, pFuncNode->node.resType.bytes, pTargetNode->slotId, pFuncNode->node.aliasName); - pExp->pExpr->_function.pFunctNode = pFuncNode; + SDataType *pType = &pFuncNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, + pType->scale, pType->precision, pFuncNode->node.aliasName); + + pExp->pExpr->_function.pFunctNode = pFuncNode; strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); // TODO: value parameter needs to be handled @@ -8092,16 +8118,18 @@ int32_t doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, return code; } - size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); - tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId); + size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, (uint64_t)queryId, taskId); code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); - pTaskInfo->pRoot = createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + pTaskInfo->pRoot = createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, + pScanPhyNode->reverse, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { - // SExchangePhysiNode* pEx = (SExchangePhysiNode*) pPhyNode; - // return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); + return createExchangeOperatorInfo(pExchange->pSrcEndPoints, pResBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. + SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableGroupInfo groupInfo = {0}; code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); @@ -8146,8 +8174,7 @@ int32_t doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, return code; } - int32_t resultRowSize = 0; - SArray* pExprInfo = createExprInfo((SAggPhysiNode*)pPhyNode, &resultRowSize); + SArray* pExprInfo = createExprInfo((SAggPhysiNode*)pPhyNode); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); pTaskInfo->pRoot = createAggregateOperatorInfo(pTaskInfo->pRoot, pExprInfo, pResBlock, pTaskInfo, pTableGroupInfo); } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index d6414a908c..c2962a3203 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -1006,7 +1006,7 @@ TEST(testCase, external_sort_Test) { #if 0 su* v = static_cast(calloc(1000000, sizeof(su))); for(int32_t i = 0; i < 1000000; ++i) { - v[i].v = rand(); + v[i].v = taosRand(); v[i].c = static_cast(malloc(4)); *(int32_t*) v[i].c = i; } @@ -1019,7 +1019,7 @@ TEST(testCase, external_sort_Test) { return; #endif - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {0}; @@ -1080,7 +1080,7 @@ TEST(testCase, external_sort_Test) { } TEST(testCase, sorted_merge_Test) { - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {0}; @@ -1152,7 +1152,7 @@ TEST(testCase, sorted_merge_Test) { } TEST(testCase, time_interval_Operator_Test) { - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {0}; diff --git a/source/libs/executor/test/lhashTests.cpp b/source/libs/executor/test/lhashTests.cpp index 66ef3b0877..88cf713727 100644 --- a/source/libs/executor/test/lhashTests.cpp +++ b/source/libs/executor/test/lhashTests.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" TEST(testCase, linear_hash_Tests) { - srand(time(NULL)); + taosSeedRand(time(NULL)); _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); #if 0 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 18d1ff41e2..7ba7d7bdcc 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -31,6 +31,12 @@ void countFunction(SqlFunctionCtx *pCtx); bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); void sumFunction(SqlFunctionCtx *pCtx); +bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +void minFunction(SqlFunctionCtx* pCtx); +void maxFunction(SqlFunctionCtx *pCtx); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b95fba9596..edb0acf075 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -41,6 +41,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = sumFunction, .finalizeFunc = functionFinalizer }, + { + .name = "min", + .type = FUNCTION_TYPE_MIN, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = minFunctionSetup, + .processFunc = minFunction, + .finalizeFunc = functionFinalizer + }, + { + .name = "max", + .type = FUNCTION_TYPE_MAX, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalizer + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, @@ -53,13 +73,43 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { } }; -const int funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); +const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { switch(pFunc->funcType) { - case FUNCTION_TYPE_COUNT: pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};break; - default: + case FUNCTION_TYPE_COUNT: + pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; break; + case FUNCTION_TYPE_SUM: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + + int32_t resType = 0; + if (IS_SIGNED_NUMERIC_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_BIGINT; + } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_UBIGINT; + } else if (IS_FLOAT_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_DOUBLE; + } else { + ASSERT(0); + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType }; + break; + } + case FUNCTION_TYPE_MIN: + case FUNCTION_TYPE_MAX: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + break; + } + case FUNCTION_TYPE_CONCAT: + // todo + break; + default: + ASSERT(0); // to found the fault ASAP. } return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fb30cce6a9..aaaee6d56c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,6 +14,7 @@ */ #include "builtinsimpl.h" +#include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" @@ -27,7 +28,6 @@ } while (0) typedef struct SSumRes { -// int8_t hasResult; union { int64_t isum; uint64_t usum; @@ -115,7 +115,7 @@ void countFunction(SqlFunctionCtx *pCtx) { } \ } while (0) -static void do_sum(SqlFunctionCtx *pCtx) { +void sumFunction(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded @@ -123,17 +123,18 @@ static void do_sum(SqlFunctionCtx *pCtx) { SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; + SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (pInput->colDataAggIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; ASSERT(numOfElem >= 0); - SSumRes* pSumInfo = (SSumRes*) pCtx->pOutput; if (IS_SIGNED_NUMERIC_TYPE(type)) { - pSumInfo->isum += pAgg->sum; + pSumRes->isum += pAgg->sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - pSumInfo->usum += pAgg->sum; + pSumRes->usum += pAgg->sum; } else if (IS_FLOAT_TYPE(type)) { - pSumInfo->dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); + pSumRes->dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); } } else { // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -141,32 +142,30 @@ static void do_sum(SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - SSumRes* pSum = (SSumRes*) pCtx->pOutput; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int8_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int16_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int32_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int64_t, numOfElem); + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_TINYINT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int8_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int16_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_INT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int32_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_BIGINT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int64_t, numOfElem); } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint8_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint16_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint32_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint64_t, numOfElem); + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_UTINYINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint8_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_USMALLINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint16_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_UINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint32_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_UBIGINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint64_t, numOfElem); } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N(pSum->dsum, pCol, start, numOfRows, double, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N(pSum->dsum, pCol, start, numOfRows, float, numOfElem); + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + LIST_ADD_N(pSumRes->dsum, pCol, start, numOfRows, double, numOfElem); + } else if (type == TSDB_DATA_TYPE_FLOAT) { + LIST_ADD_N(pSumRes->dsum, pCol, start, numOfRows, float, numOfElem); } } @@ -179,14 +178,271 @@ bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -void sumFunction(SqlFunctionCtx *pCtx) { - do_sum(pCtx); +bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } - // keep the result data in output buffer, not in the intermediate buffer -// SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult == DATA_SET_FLAG) { - // set the flag for super table query -// SSumRes *pSum = (SSumRes *)pCtx->pOutput; -// pSum->hasResult = DATA_SET_FLAG; -// } + char* buf = GET_ROWCELL_INTERBUF(pResultInfo); + switch (pCtx->resDataInfo.type) { + case TSDB_DATA_TYPE_INT: + *((int32_t *)buf) = INT32_MIN; + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)buf) = 0; + break; + case TSDB_DATA_TYPE_FLOAT: + *((float *)buf) = -FLT_MAX; + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(((double *)buf), -DBL_MAX); + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)buf) = INT64_MIN; + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)buf) = 0; + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)buf) = INT16_MIN; + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)buf) = 0; + break; + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)buf) = INT8_MIN; + break; + case TSDB_DATA_TYPE_UTINYINT: + *((uint8_t *)buf) = 0; + break; + default: + assert(0); + } + return true; } + +bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; // not initialized since it has been initialized + } + + char* buf = GET_ROWCELL_INTERBUF(pResultInfo); + switch (pCtx->resDataInfo.type) { + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)buf) = INT8_MAX; + break; + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *) buf = UINT8_MAX; + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)buf) = INT16_MAX; + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)buf) = UINT16_MAX; + break; + case TSDB_DATA_TYPE_INT: + *((int32_t *)buf) = INT32_MAX; + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)buf) = UINT32_MAX; + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)buf) = INT64_MAX; + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)buf) = UINT64_MAX; + break; + case TSDB_DATA_TYPE_FLOAT: + *((float *)buf) = FLT_MAX; + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(((double *)buf), DBL_MAX); + break; + default: + assert(0); + } + + return true; +} + +bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); + pEnv->calcMemSize = sizeof(int64_t); + return true; +} + +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) + +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0); + +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0) + +#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_SUBSID_RES(ctx, _ts); \ + (num) += 1; \ + } \ + } while (0) + +#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ + do { \ + _t* d = (_t*)((_col)->pData); \ + for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + } \ + TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \ + UPDATE_DATA(ctx, val, d[i], num, sign, ts); \ + } \ + } while (0) + +int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { + int32_t numOfElems = 0; + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + + SColumnInfoData* pCol = pInput->pData[0]; + int32_t type = pCol->info.type; + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + + // data in current data block are qualified to the query + if (pInput->colDataAggIsSet) { + numOfElems = pInput->numOfRows - pAgg->numOfNull; + ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0); + + if (numOfElems == 0) { + return numOfElems; + } + + void* tval = NULL; + int16_t index = 0; + + if (isMinFunc) { + tval = &pInput->pColumnDataAgg[0]->min; + index = pInput->pColumnDataAgg[0]->minIndex; + } else { + tval = &pInput->pColumnDataAgg[0]->max; + index = pInput->pColumnDataAgg[0]->maxIndex; + } + + TSKEY key = TSKEY_INITIAL_VAL; + if (pCtx->ptsList != NULL) { + // the index is the original position, not the relative position + key = pCtx->ptsList[index]; + } + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + int64_t val = GET_INT64_VAL(tval); + +#if defined(_DEBUG_VIEW) + qDebug("max value updated according to pre-cal:%d", *data); +#endif + + if ((*(int64_t*)buf < val) ^ isMinFunc) { + *(int64_t*) buf = val; + for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor + __ctx->tag.i = key; + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; + } + + __ctx->fpSet.process(__ctx); + } + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + uint64_t val = GET_UINT64_VAL(tval); + UPDATE_DATA(pCtx, *(uint64_t*)buf, val, numOfElems, isMinFunc, key); + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = GET_DOUBLE_VAL(tval); + UPDATE_DATA(pCtx, *(double*)buf, val, numOfElems, isMinFunc, key); + } else if (type == TSDB_DATA_TYPE_FLOAT) { + double val = GET_DOUBLE_VAL(tval); + UPDATE_DATA(pCtx, *(float*)buf, (float)val, numOfElems, isMinFunc, key); + } + + return numOfElems; + } + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_TINYINT) { + LOOPCHECK_N(*(int8_t*)buf, pCol, pCtx, int8_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + LOOPCHECK_N(*(int16_t*) buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_INT) { + int32_t *pData = (int32_t*)pCol->pData; + int32_t *val = (int32_t*) buf; + + for (int32_t i = 0; i < pCtx->size; ++i) { + if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + if ((*val < pData[i]) ^ isMinFunc) { + *val = pData[i]; + TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i) : 0; + DO_UPDATE_SUBSID_RES(pCtx, ts); + } + + numOfElems += 1; + } + +#if defined(_DEBUG_VIEW) + qDebug("max value updated:%d", *retVal); +#endif + } else if (type == TSDB_DATA_TYPE_BIGINT) { + LOOPCHECK_N(*(int64_t*) buf, pCol, pCtx, int64_t, numOfRows, start, isMinFunc, numOfElems); + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_UTINYINT) { + LOOPCHECK_N(*(uint8_t*) buf, pCol, pCtx, uint8_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_USMALLINT) { + LOOPCHECK_N(*(uint16_t*) buf, pCol, pCtx, uint16_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_UINT) { + LOOPCHECK_N(*(uint32_t*) buf, pCol, pCtx, uint32_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_UBIGINT) { + LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint64_t, numOfRows, start, isMinFunc, numOfElems); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + LOOPCHECK_N(*(double*) buf, pCol, pCtx, double, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_FLOAT) { + LOOPCHECK_N(*(float*) buf, pCol, pCtx, float, numOfRows, start, isMinFunc, numOfElems); + } + + return numOfElems; +} + +void minFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = doMinMaxHelper(pCtx, 1); + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); +} + +void maxFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = doMinMaxHelper(pCtx, 0); + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); +} \ No newline at end of file diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 47d09ec2dc..4360515328 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -958,157 +958,6 @@ static void avg_finalizer(SqlFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// -static void minMax_function(SqlFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - // data in current data block are qualified to the query - if (pCtx->isAggSet) { - *notNullElems = pCtx->size - pCtx->agg.numOfNull; - assert(*notNullElems >= 0); - - if (*notNullElems == 0) { - return; - } - - void* tval = NULL; - int16_t index = 0; - - if (isMin) { - tval = &pCtx->agg.min; - index = pCtx->agg.minIndex; - } else { - tval = &pCtx->agg.max; - index = pCtx->agg.maxIndex; - } - - TSKEY key = TSKEY_INITIAL_VAL; - if (pCtx->ptsList != NULL) { - /** - * NOTE: work around the bug caused by invalid pre-calculated function. - * Here the selectivity + ts will not return correct value. - * - * The following codes of 3 lines will be removed later. - */ -// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { -// index = 0; -// } - - // the index is the original position, not the relative position - key = pCtx->ptsList[index]; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t val = GET_INT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *data = (int8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *data = (int16_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *data = (int32_t *)pOutput; -#if defined(_DEBUG_VIEW) - qDebug("max value updated according to pre-cal:%d", *data); -#endif - - if ((*data < val) ^ isMin) { - *data = (int32_t)val; - for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; - if (__ctx->functionId == FUNCTION_TS_DUMMY) { - __ctx->tag.i = key; - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - } - - aggFunc[FUNCTION_TAG].addInput(__ctx); - } - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *data = (int64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t val = GET_UINT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - uint8_t *data = (uint8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (uint8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - uint16_t *data = (uint16_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - uint32_t *data = (uint32_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint32_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - uint64_t *data = (uint64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *data = (double *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *data = (float *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, (float)val, notNullElems, isMin, key); - } - - return; - } - - void *p = GET_INPUT_DATA_LIST(pCtx); - TSKEY *tsList = GET_TS_LIST(pCtx); - - *notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pData = p; - int32_t *retVal = (int32_t*) pOutput; - - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { - continue; - } - - if ((*retVal < pData[i]) ^ isMin) { - *retVal = pData[i]; - TSKEY k = tsList[i]; - - DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - *notNullElems += 1; - } -#if defined(_DEBUG_VIEW) - qDebug("max value updated:%d", *retVal); -#endif - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - TYPED_LOOPCHECK_N(uint8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LOOPCHECK_N(float, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } -} - static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized @@ -1204,43 +1053,9 @@ static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf /* * the output result of min/max function is the final output buffer, not the intermediate result buffer */ -static void min_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - -static void max_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; - +#if 0 GET_TRUE_DATA_TYPE(); assert(pCtx->stableQuery); @@ -1319,7 +1134,8 @@ static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *outp break; } } - +#endif + return notNullElems; } @@ -1618,7 +1434,7 @@ static void first_function(SqlFunctionCtx *pCtx) { memcpy(pCtx->pOutput, data, pCtx->inputBytes); if (pCtx->ptsList != NULL) { TSKEY k = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, k); +// DO_UPDATE_TAG_COLUMNS(pCtx, k); } SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); @@ -1642,7 +1458,7 @@ static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t in pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1696,7 +1512,7 @@ static void first_dist_func_merge(SqlFunctionCtx *pCtx) { pCtx->param[1].i = pInput->ts; pCtx->param[1].nType = pCtx->resDataInfo.type; - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } SET_VAL(pCtx, 1, 1); @@ -1730,7 +1546,7 @@ static void last_function(SqlFunctionCtx *pCtx) { memcpy(pCtx->pOutput, data, pCtx->inputBytes); TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); //pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed on this column @@ -1777,7 +1593,7 @@ static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t ind pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1833,7 +1649,7 @@ static void last_dist_func_merge(SqlFunctionCtx *pCtx) { pCtx->param[1].i = pInput->ts; pCtx->param[1].nType = pCtx->resDataInfo.type; - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } SET_VAL(pCtx, 1, 1); @@ -1860,10 +1676,10 @@ static void last_row_function(SqlFunctionCtx *pCtx) { pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } else { TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); } SET_VAL(pCtx, pCtx->size, 1); @@ -1883,25 +1699,25 @@ static void last_row_finalizer(SqlFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SExtTagsInfo *pTagInfo, int16_t stage) { + SSubsidiaryResInfo *pTagInfo, int16_t stage) { dst->v.nType = type; dst->v.i = *(int64_t *)val; dst->timestamp = tsKey; int32_t size = 0; if (stage == MERGE_STAGE) { - memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); +// memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields - for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; - if (ctx->functionId == FUNCTION_TS_DUMMY) { - ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - ctx->tag.i = tsKey; - } - - taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); - size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; - } +// for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { +// SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; +// if (ctx->functionId == FUNCTION_TS_DUMMY) { +// ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; +// ctx->tag.i = tsKey; +// } +// +// taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); +// size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; +// } } } @@ -1956,7 +1772,7 @@ static void topBotSwapFn(void *dst, void *src, const void *param) } static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { + SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1966,7 +1782,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); +// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); pInfo->num++; } else { @@ -1974,13 +1790,13 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pList[0]->v.u) || (IS_FLOAT_TYPE(type) && val.d > pList[0]->v.d)) { valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); +// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); } } } static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { + SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1990,7 +1806,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); +// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); pInfo->num++; } else { @@ -1998,7 +1814,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u < pList[0]->v.u) || (IS_FLOAT_TYPE(type) && val.d < pList[0]->v.d)) { valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); +// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); } } } @@ -2113,21 +1929,21 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // set the corresponding tag data for each record // todo check malloc failure - char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); - for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; - } +// char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); +// for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { +// pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; +// } - for (int32_t i = 0; i < len; ++i, output += step) { - int16_t offset = 0; - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); - offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; - pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; - } - } +// for (int32_t i = 0; i < len; ++i, output += step) { +// int16_t offset = 0; +// for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { +// memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); +// offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; +// pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; +// } +// } - tfree(pData); +// tfree(pData); } /* @@ -2161,13 +1977,13 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i; - size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; +// size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - for (int32_t i = 0; i < pCtx->param[0].i; ++i) { - pTopBotInfo->res[i] = (tValuePair*) tmp; - pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); - tmp += size; - } +// for (int32_t i = 0; i < pCtx->param[0].i; ++i) { +// pTopBotInfo->res[i] = (tValuePair*) tmp; +// pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); +// tmp += size; +// } } bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { @@ -2256,7 +2072,7 @@ static void top_function(SqlFunctionCtx *pCtx) { // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_top_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2283,8 +2099,8 @@ static void top_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, - type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); +// do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, +// type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } SET_VAL(pCtx, pInput->num, pOutput->num); @@ -2313,7 +2129,7 @@ static void bottom_function(SqlFunctionCtx *pCtx) { notNullElems++; // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2340,8 +2156,8 @@ static void bottom_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, - &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); +// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, +// &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } SET_VAL(pCtx, pInput->num, pOutput->num); @@ -4448,7 +4264,7 @@ SAggFunctionInfo aggFunc[35] = {{ FUNCTION_MIN, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, min_func_setup, - min_function, + NULL, function_finalizer, min_func_merge, statisRequired, @@ -4461,7 +4277,7 @@ SAggFunctionInfo aggFunc[35] = {{ FUNCTION_MAX, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, max_func_setup, - max_function, + NULL, function_finalizer, max_func_merge, statisRequired, diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index fd267fbf03..89f3f8ba8a 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -722,13 +722,13 @@ static SArray* tfileGetFileList(const char* path) { uint32_t version; SArray* files = taosArrayInit(4, sizeof(void*)); - DIR* dir = opendir(path); - if (NULL == dir) { + TdDirPtr pDir = taosOpenDir(path); + if (NULL == pDir) { return NULL; } - struct dirent* entry; - while ((entry = readdir(dir)) != NULL) { - char* file = entry->d_name; + TdDirEntryPtr pDirEntry; + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char* file = taosGetDirEntryName(pDirEntry); if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } @@ -738,7 +738,7 @@ static SArray* tfileGetFileList(const char* path) { sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } - closedir(dir); + taosCloseDir(pDir); taosArraySort(files, tfileCompare); tfileRmExpireFile(files); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index ce3f7fe25e..699c785be5 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -699,7 +699,7 @@ class IndexObj { for (int i = 0; i < numOfTable; i++) { for (int k = 0; k < 10 && k < colVal.size(); k++) { // opt - tColVal[rand() % colValSize] = 'a' + k % 26; + tColVal[taosRand() % colValSize] = 'a' + k % 26; } SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 050372fab3..58f1c08039 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -3,7 +3,6 @@ add_library(monitor STATIC ${MONITOR_SRC}) target_include_directories( monitor PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/monitor" - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index c3b6569555..6ef901410b 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -19,7 +19,6 @@ #include "monitor.h" #include "tarray.h" -#include "tlockfree.h" #include "tjson.h" typedef struct { diff --git a/source/libs/nodes/inc/nodesint.h b/source/libs/nodes/inc/nodesUtil.h similarity index 100% rename from source/libs/nodes/inc/nodesint.h rename to source/libs/nodes/inc/nodesUtil.h diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 27d71ff532..a60366c0d6 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodesint.h" +#include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" #include "taos.h" @@ -60,6 +60,9 @@ #define CLONE_OBJECT_FIELD(fldname, cloneFunc) \ do { \ + if (NULL == (pSrc)->fldname) { \ + break; \ + } \ (pDst)->fldname = cloneFunc((pSrc)->fldname); \ if (NULL == (pDst)->fldname) { \ nodesDestroyNode((SNode*)(pDst)); \ @@ -234,10 +237,17 @@ static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* } static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_SCALAR_FIELD(msgType); return (SNode*)pDst; } +static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(srcGroupId); + return (SNode*)pDst; +} + static SNode* logicSubplanCopy(const SSubLogicPlan* pSrc, SSubLogicPlan* pDst) { CLONE_NODE_FIELD(pNode); COPY_SCALAR_FIELD(subplanType); @@ -261,6 +271,13 @@ static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) { return (SNode*)pDst; } +static SNode* downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) { + COPY_SCALAR_FIELD(addr); + COPY_SCALAR_FIELD(taskId); + COPY_SCALAR_FIELD(schedId); + return (SNode*)pDst; +} + SNodeptr nodesCloneNode(const SNodeptr pNode) { if (NULL == pNode) { return NULL; @@ -296,6 +313,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return dataBlockDescCopy((const SDataBlockDescNode*)pNode, (SDataBlockDescNode*)pDst); case QUERY_NODE_SLOT_DESC: return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst); + case QUERY_NODE_DOWNSTREAM_SOURCE: + return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanCopy((const SScanLogicNode*)pNode, (SScanLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_AGG: @@ -304,6 +323,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicProjectCopy((const SProjectLogicNode*)pNode, (SProjectLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: return logicVnodeModifCopy((const SVnodeModifLogicNode*)pNode, (SVnodeModifLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanCopy((const SSubLogicPlan*)pNode, (SSubLogicPlan*)pDst); default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b28c45e554..875f19b660 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodesint.h" +#include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" #include "query.h" @@ -496,6 +496,37 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkExchangePhysiPlanSrcGroupId = "SrcGroupId"; +static const char* jkExchangePhysiPlanSrcEndPoints = "SrcEndPoints"; + +static int32_t physiExchangeNodeToJson(const void* pObj, SJson* pJson) { + const SExchangePhysiNode* pNode = (const SExchangePhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkExchangePhysiPlanSrcGroupId, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkExchangePhysiPlanSrcEndPoints, pNode->pSrcEndPoints); + } + + return code; +} + +static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { + SExchangePhysiNode* pNode = (SExchangePhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkExchangePhysiPlanSrcGroupId, &pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkExchangePhysiPlanSrcEndPoints, &pNode->pSrcEndPoints); + } + + return code; +} + static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { @@ -517,7 +548,7 @@ static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { } static const char* jkSubplanIdQueryId = "QueryId"; -static const char* jkSubplanIdTemplateId = "TemplateId"; +static const char* jkSubplanIdGroupId = "GroupId"; static const char* jkSubplanIdSubplanId = "SubplanId"; static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { @@ -525,7 +556,7 @@ static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkSubplanIdQueryId, pNode->queryId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSubplanIdTemplateId, pNode->templateId); + code = tjsonAddIntegerToObject(pJson, jkSubplanIdGroupId, pNode->groupId); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanIdSubplanId, pNode->subplanId); @@ -539,7 +570,7 @@ static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { int32_t code = tjsonGetUBigIntValue(pJson, jkSubplanIdQueryId, &pNode->queryId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkSubplanIdTemplateId, &pNode->templateId); + code = tjsonGetIntValue(pJson, jkSubplanIdGroupId, &pNode->groupId); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanIdSubplanId, &pNode->subplanId); @@ -583,13 +614,13 @@ static int32_t queryNodeAddrToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkQueryNodeAddrId, pNode->nodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkQueryNodeAddrInUse, pNode->epset.inUse); + code = tjsonAddIntegerToObject(pJson, jkQueryNodeAddrInUse, pNode->epSet.inUse); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkQueryNodeAddrNumOfEps, pNode->epset.numOfEps); + code = tjsonAddIntegerToObject(pJson, jkQueryNodeAddrNumOfEps, pNode->epSet.numOfEps); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddArray(pJson, jkQueryNodeAddrEps, epToJson, pNode->epset.eps, sizeof(SEp), pNode->epset.numOfEps); + code = tjsonAddArray(pJson, jkQueryNodeAddrEps, epToJson, pNode->epSet.eps, sizeof(SEp), pNode->epSet.numOfEps); } return code; @@ -600,13 +631,13 @@ static int32_t jsonToQueryNodeAddr(const SJson* pJson, void* pObj) { int32_t code = tjsonGetIntValue(pJson, jkQueryNodeAddrId, &pNode->nodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkQueryNodeAddrInUse, &pNode->epset.inUse); + code = tjsonGetTinyIntValue(pJson, jkQueryNodeAddrInUse, &pNode->epSet.inUse); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkQueryNodeAddrNumOfEps, &pNode->epset.numOfEps); + code = tjsonGetTinyIntValue(pJson, jkQueryNodeAddrNumOfEps, &pNode->epSet.numOfEps); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToArray(pJson, jkQueryNodeAddrEps, jsonToEp, pNode->epset.eps, sizeof(SEp)); + code = tjsonToArray(pJson, jkQueryNodeAddrEps, jsonToEp, pNode->epSet.eps, sizeof(SEp)); } return code; @@ -1230,6 +1261,38 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkDownstreamSourceAddr = "Addr"; +static const char* jkDownstreamSourceTaskId = "TaskId"; +static const char* jkDownstreamSourceSchedId = "SchedId"; + +static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { + const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkDownstreamSourceAddr, queryNodeAddrToJson, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceTaskId, pNode->taskId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceSchedId, pNode->schedId); + } + + return code; +} + +static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)pObj; + + int32_t code = tjsonToObject(pJson, jkDownstreamSourceAddr, jsonToQueryNodeAddr, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceTaskId, &pNode->taskId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceSchedId, &pNode->schedId); + } + + return code; +} + static const char* jkDataBlockDescDataBlockId = "DataBlockId"; static const char* jkDataBlockDescSlots = "Slots"; static const char* jkDataBlockResultRowSize = "ResultRowSize"; @@ -1350,6 +1413,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SLOT_DESC: return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_COLUMN_DEF: + break; + case QUERY_NODE_DOWNSTREAM_SOURCE: + return downstreamSourceNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: break; case QUERY_NODE_SELECT_STMT: @@ -1387,7 +1453,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_AGG: return physiAggNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + return physiExchangeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: + break; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -1437,6 +1505,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDataBlockDescNode(pJson, pObj); case QUERY_NODE_SLOT_DESC: return jsonToSlotDescNode(pJson, pObj); + case QUERY_NODE_DOWNSTREAM_SOURCE: + return jsonToDownstreamSourceNode(pJson, pObj); // case QUERY_NODE_SET_OPERATOR: // break; // case QUERY_NODE_SELECT_STMT: @@ -1459,6 +1529,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiJoinNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_AGG: return jsonToPhysiAggNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + return jsonToPhysiExchangeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 39e64ba107..dc533d1423 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -14,7 +14,7 @@ */ #include "cmdnodes.h" -#include "nodesint.h" +#include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" #include "taos.h" @@ -74,6 +74,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSlotDescNode)); case QUERY_NODE_COLUMN_DEF: return makeNode(type, sizeof(SColumnDefNode)); + case QUERY_NODE_DOWNSTREAM_SOURCE: + return makeNode(type, sizeof(SDownstreamSourceNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: @@ -129,6 +131,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SProjectLogicNode)); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: return makeNode(type, sizeof(SVnodeModifLogicNode)); + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: return makeNode(type, sizeof(SSubLogicPlan)); case QUERY_NODE_LOGIC_PLAN: diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y deleted file mode 100644 index 6e09048810..0000000000 --- a/source/libs/parser/inc/new_sql.y +++ /dev/null @@ -1,557 +0,0 @@ -//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql -//usage: lemon sql.y - -%name NewParse - -%token_prefix TK_ -%token_type { SToken } -%default_type { SNode* } -%default_destructor { nodesDestroyNode($$); } - -%extra_argument { SAstCreateContext* pCxt } - -%include { -#include -#include -#include -#include -#include - -#include "nodes.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "astCreateFuncs.h" -} - -%syntax_error { - if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); - } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); - } - pCxt->valid = false; -} - -%left OR. -%left AND. -//%right NOT. -%left UNION ALL MINUS EXCEPT INTERSECT. -%left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. -%left NK_PLUS NK_MINUS. -//%left DIVIDE TIMES. -%left NK_STAR NK_SLASH NK_REM. -%left NK_CONCAT. -//%right NK_BITNOT. - -/************************************************ create/alter/drop/show user *****************************************/ -cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B);} -cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B);} -cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B);} -cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } -cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } - -/************************************************ create/drop/show dnode **********************************************/ -cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL);} -cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B);} -cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} -cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} -cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } - -%type dnode_endpoint { SToken } -%destructor dnode_endpoint { } -dnode_endpoint(A) ::= NK_STRING(B). { A = B; } - -%type dnode_host_name { SToken } -%destructor dnode_host_name { } -dnode_host_name(A) ::= NK_ID(B). { A = B; } -dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } - -/************************************************ create/drop/show/use database ***************************************/ -cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C);} -cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } -cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } -cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A);} - -%type not_exists_opt { bool } -%destructor not_exists_opt { } -not_exists_opt(A) ::= IF NOT EXISTS. { A = true; } -not_exists_opt(A) ::= . { A = false; } - -%type exists_opt { bool } -%destructor exists_opt { } -exists_opt(A) ::= IF EXISTS. { A = true; } -exists_opt(A) ::= . { A = false; } - -%type db_options { SDatabaseOptions* } -%destructor db_options { tfree($$); } -db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } -db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BLOCKS, &C); } -db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHE, &C); } -db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } -db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } -db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } -db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); } -db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); } -db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); } -db_options(A) ::= db_options(B) KEEP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, &C); } -db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } -db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); } -db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } -db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } -db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } -db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } -db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } -db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } - -/************************************************ create/drop/show table/stable ***************************************/ -cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} -cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} -cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} -cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } -cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } -cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } -cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } - -%type multi_create_clause { SNodeList* } -%destructor multi_create_clause { nodesDestroyList($$); } -multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } -multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); } - -create_subtable_clause(A) ::= - not_exists_opt(B) full_table_name(C) USING full_table_name(D) - specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP. { A = createCreateSubTableClause(pCxt, B, C, D, E, F); } - -%type multi_drop_clause { SNodeList* } -%destructor multi_drop_clause { nodesDestroyList($$); } -multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); } -multi_drop_clause(A) ::= multi_drop_clause(B) drop_table_clause(C). { A = addNodeToList(pCxt, B, C); } - -drop_table_clause(A) ::= exists_opt(B) full_table_name(C). { A = createDropTableClause(pCxt, B, C); } - -%type specific_tags_opt { SNodeList* } -%destructor specific_tags_opt { nodesDestroyList($$); } -specific_tags_opt(A) ::= . { A = NULL; } -specific_tags_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } - -full_table_name(A) ::= table_name(B). { A = createRealTableNode(pCxt, NULL, &B, NULL); } -full_table_name(A) ::= db_name(B) NK_DOT table_name(C). { A = createRealTableNode(pCxt, &B, &C, NULL); } - -%type column_def_list { SNodeList* } -%destructor column_def_list { nodesDestroyList($$); } -column_def_list(A) ::= column_def(B). { A = createNodeList(pCxt, B); } -column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } - -column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } -column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } - -%type type_name { SDataType } -%destructor type_name { } -type_name(A) ::= BOOL. { A = createDataType(TSDB_DATA_TYPE_BOOL); } -type_name(A) ::= TINYINT. { A = createDataType(TSDB_DATA_TYPE_TINYINT); } -type_name(A) ::= SMALLINT. { A = createDataType(TSDB_DATA_TYPE_SMALLINT); } -type_name(A) ::= INT. { A = createDataType(TSDB_DATA_TYPE_INT); } -type_name(A) ::= INTEGER. { A = createDataType(TSDB_DATA_TYPE_INT); } -type_name(A) ::= BIGINT. { A = createDataType(TSDB_DATA_TYPE_BIGINT); } -type_name(A) ::= FLOAT. { A = createDataType(TSDB_DATA_TYPE_FLOAT); } -type_name(A) ::= DOUBLE. { A = createDataType(TSDB_DATA_TYPE_DOUBLE); } -type_name(A) ::= BINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &B); } -type_name(A) ::= TIMESTAMP. { A = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } -type_name(A) ::= NCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &B); } -type_name(A) ::= TINYINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UTINYINT); } -type_name(A) ::= SMALLINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_USMALLINT); } -type_name(A) ::= INT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UINT); } -type_name(A) ::= BIGINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UBIGINT); } -type_name(A) ::= JSON. { A = createDataType(TSDB_DATA_TYPE_JSON); } -type_name(A) ::= VARCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &B); } -type_name(A) ::= MEDIUMBLOB. { A = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } -type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } -type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } -type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } - -%type tags_def_opt { SNodeList* } -%destructor tags_def_opt { nodesDestroyList($$); } -tags_def_opt(A) ::= . { A = NULL; } -tags_def_opt(A) ::= tags_def(B). { A = B; } - -%type tags_def { SNodeList* } -%destructor tags_def { nodesDestroyList($$); } -tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } - -%type table_options { STableOptions* } -%destructor table_options { tfree($$); } -table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} -table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } -table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } -table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } -table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } - -%type col_name_list { SNodeList* } -%destructor col_name_list { nodesDestroyList($$); } -col_name_list(A) ::= col_name(B). { A = createNodeList(pCxt, B); } -col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } - -col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } - -/************************************************ show vgroups ********************************************************/ -cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } -cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); } - -/************************************************ select **************************************************************/ -cmd ::= query_expression(A). { pCxt->pRootNode = A; } - -/************************************************ literal *************************************************************/ -literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } -literal(A) ::= NK_FLOAT(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); } -literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)); } -literal(A) ::= NK_BOOL(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); } -literal(A) ::= TIMESTAMP(B) NK_STRING(C). { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); } -literal(A) ::= duration_literal(B). { A = B; } - -duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } - -%type literal_list { SNodeList* } -%destructor literal_list { nodesDestroyList($$); } -literal_list(A) ::= literal(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } - -/************************************************ names and identifiers ***********************************************/ -%type db_name { SToken } -%destructor db_name { } -db_name(A) ::= NK_ID(B). { A = B; } - -%type table_name { SToken } -%destructor table_name { } -table_name(A) ::= NK_ID(B). { A = B; } - -%type column_name { SToken } -%destructor column_name { } -column_name(A) ::= NK_ID(B). { A = B; } - -%type function_name { SToken } -%destructor function_name { } -function_name(A) ::= NK_ID(B). { A = B; } - -%type table_alias { SToken } -%destructor table_alias { } -table_alias(A) ::= NK_ID(B). { A = B; } - -%type column_alias { SToken } -%destructor column_alias { } -column_alias(A) ::= NK_ID(B). { A = B; } - -%type user_name { SToken } -%destructor user_name { } -user_name(A) ::= NK_ID(B). { A = B; } - -/************************************************ expression **********************************************************/ -expression(A) ::= literal(B). { A = B; } -//expression(A) ::= NK_QUESTION(B). { A = B; } -//expression(A) ::= pseudo_column(B). { A = B; } -expression(A) ::= column_reference(B). { A = B; } -expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } -expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); } -//expression(A) ::= cast_expression(B). { A = B; } -//expression(A) ::= case_expression(B). { A = B; } -expression(A) ::= subquery(B). { A = B; } -expression(A) ::= NK_LP(B) expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, releaseRawExprNode(pCxt, C)); } -expression(A) ::= NK_PLUS(B) expression(C). { - SToken t = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &B, &t, releaseRawExprNode(pCxt, C)); - } -expression(A) ::= NK_MINUS(B) expression(C). { - SToken t = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL)); - } -expression(A) ::= expression(B) NK_PLUS expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_MINUS expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_STAR expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_SLASH expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_REM expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } - -%type expression_list { SNodeList* } -%destructor expression_list { nodesDestroyList($$); } -expression_list(A) ::= expression(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } - -column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } -column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } - -//pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); } - -/************************************************ predicate ***********************************************************/ -predicate(A) ::= expression(B) compare_op(C) expression(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); - } -//predicate(A) ::= expression(B) compare_op sub_type expression(B). -predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); - } -predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); - } -predicate(A) ::= expression(B) IS NULL(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, B), NULL)); - } -predicate(A) ::= expression(B) IS NOT NULL(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, B), NULL)); - } -predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); - } - -%type compare_op { EOperatorType } -%destructor compare_op { } -compare_op(A) ::= NK_LT. { A = OP_TYPE_LOWER_THAN; } -compare_op(A) ::= NK_GT. { A = OP_TYPE_GREATER_THAN; } -compare_op(A) ::= NK_LE. { A = OP_TYPE_LOWER_EQUAL; } -compare_op(A) ::= NK_GE. { A = OP_TYPE_GREATER_EQUAL; } -compare_op(A) ::= NK_NE. { A = OP_TYPE_NOT_EQUAL; } -compare_op(A) ::= NK_EQ. { A = OP_TYPE_EQUAL; } -compare_op(A) ::= LIKE. { A = OP_TYPE_LIKE; } -compare_op(A) ::= NOT LIKE. { A = OP_TYPE_NOT_LIKE; } -compare_op(A) ::= MATCH. { A = OP_TYPE_MATCH; } -compare_op(A) ::= NMATCH. { A = OP_TYPE_NMATCH; } - -%type in_op { EOperatorType } -%destructor in_op { } -in_op(A) ::= IN. { A = OP_TYPE_IN; } -in_op(A) ::= NOT IN. { A = OP_TYPE_NOT_IN; } - -in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } - -/************************************************ boolean_value_expression ********************************************/ -boolean_value_expression(A) ::= boolean_primary(B). { A = B; } -boolean_value_expression(A) ::= NOT(C) boolean_primary(B). { - SToken e = getTokenFromRawExprNode(pCxt, B); - A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); - } -boolean_value_expression(A) ::= - boolean_value_expression(B) OR boolean_value_expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -boolean_value_expression(A) ::= - boolean_value_expression(B) AND boolean_value_expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } - -boolean_primary(A) ::= predicate(B). { A = B; } -boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, releaseRawExprNode(pCxt, B)); } - -/************************************************ common_expression ********************************************/ -common_expression(A) ::= expression(B). { A = B; } -common_expression(A) ::= boolean_value_expression(B). { A = B; } - -/************************************************ from_clause *********************************************************/ -from_clause(A) ::= FROM table_reference_list(B). { A = B; } - -table_reference_list(A) ::= table_reference(B). { A = B; } -table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } - -/************************************************ table_reference *****************************************************/ -table_reference(A) ::= table_primary(B). { A = B; } -table_reference(A) ::= joined_table(B). { A = B; } - -table_primary(A) ::= table_name(B) alias_opt(C). { A = createRealTableNode(pCxt, NULL, &B, &C); } -table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { A = createRealTableNode(pCxt, &B, &C, &D); } -table_primary(A) ::= subquery(B) alias_opt(C). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), &C); } -table_primary(A) ::= parenthesized_joined_table(B). { A = B; } - -%type alias_opt { SToken } -%destructor alias_opt { } -alias_opt(A) ::= . { A = nil_token; } -alias_opt(A) ::= table_alias(B). { A = B; } -alias_opt(A) ::= AS table_alias(B). { A = B; } - -parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { A = B; } -parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { A = B; } - -/************************************************ joined_table ********************************************************/ -joined_table(A) ::= - table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { A = createJoinTableNode(pCxt, C, B, D, E); } - -%type join_type { EJoinType } -%destructor join_type { } -join_type(A) ::= . { A = JOIN_TYPE_INNER; } -join_type(A) ::= INNER. { A = JOIN_TYPE_INNER; } - -/************************************************ query_specification *************************************************/ -query_specification(A) ::= - SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) - partition_by_clause_opt(F) twindow_clause_opt(G) - group_by_clause_opt(H) having_clause_opt(I). { - A = createSelectStmt(pCxt, B, C, D); - A = addWhereClause(pCxt, A, E); - A = addPartitionByClause(pCxt, A, F); - A = addWindowClauseClause(pCxt, A, G); - A = addGroupByClause(pCxt, A, H); - A = addHavingClause(pCxt, A, I); - } - -%type set_quantifier_opt { bool } -%destructor set_quantifier_opt { } -set_quantifier_opt(A) ::= . { A = false; } -set_quantifier_opt(A) ::= DISTINCT. { A = true; } -set_quantifier_opt(A) ::= ALL. { A = false; } - -%type select_list { SNodeList* } -%destructor select_list { nodesDestroyList($$); } -select_list(A) ::= NK_STAR. { A = NULL; } -select_list(A) ::= select_sublist(B). { A = B; } - -%type select_sublist { SNodeList* } -%destructor select_sublist { nodesDestroyList($$); } -select_sublist(A) ::= select_item(B). { A = createNodeList(pCxt, B); } -select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { A = addNodeToList(pCxt, B, C); } - -select_item(A) ::= common_expression(B). { - SToken t = getTokenFromRawExprNode(pCxt, B); - A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t); - } -select_item(A) ::= common_expression(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } -select_item(A) ::= common_expression(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } -select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { A = createColumnNode(pCxt, &B, &C); } - -where_clause_opt(A) ::= . { A = NULL; } -where_clause_opt(A) ::= WHERE search_condition(B). { A = B; } - -%type partition_by_clause_opt { SNodeList* } -%destructor partition_by_clause_opt { nodesDestroyList($$); } -partition_by_clause_opt(A) ::= . { A = NULL; } -partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { A = B; } - -twindow_clause_opt(A) ::= . { A = NULL; } -twindow_clause_opt(A) ::= - SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } -twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } -twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, B, NULL, C, D); } -twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP - sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, B, C, D, E); } - -sliding_opt(A) ::= . { A = NULL; } -sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = B; } - -fill_opt(A) ::= . { A = NULL; } -fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } -fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } - -%type fill_mode { EFillMode } -%destructor fill_mode { } -fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; } -fill_mode(A) ::= PREV. { A = FILL_MODE_PREV; } -fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; } -fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; } -fill_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; } - -%type group_by_clause_opt { SNodeList* } -%destructor group_by_clause_opt { nodesDestroyList($$); } -group_by_clause_opt(A) ::= . { A = NULL; } -group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { A = B; } - -%type group_by_list { SNodeList* } -%destructor group_by_list { nodesDestroyList($$); } -group_by_list(A) ::= expression(B). { A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); } -group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); } - -having_clause_opt(A) ::= . { A = NULL; } -having_clause_opt(A) ::= HAVING search_condition(B). { A = B; } - -/************************************************ query_expression ****************************************************/ -query_expression(A) ::= - query_expression_body(B) - order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { - A = addOrderByClause(pCxt, B, C); - A = addSlimitClause(pCxt, A, D); - A = addLimitClause(pCxt, A, E); - } - -query_expression_body(A) ::= query_primary(B). { A = B; } -query_expression_body(A) ::= - query_expression_body(B) UNION ALL query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } - -query_primary(A) ::= query_specification(B). { A = B; } -//query_primary(A) ::= -// NK_LP query_expression_body(B) -// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} - -%type order_by_clause_opt { SNodeList* } -%destructor order_by_clause_opt { nodesDestroyList($$); } -order_by_clause_opt(A) ::= . { A = NULL; } -order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { A = B; } - -slimit_clause_opt(A) ::= . { A = NULL; } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } - -limit_clause_opt(A) ::= . { A = NULL; } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } - -/************************************************ subquery ************************************************************/ -subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, C); } - -/************************************************ search_condition ****************************************************/ -search_condition(A) ::= common_expression(B). { A = releaseRawExprNode(pCxt, B); } - -/************************************************ sort_specification_list *********************************************/ -%type sort_specification_list { SNodeList* } -%destructor sort_specification_list { nodesDestroyList($$); } -sort_specification_list(A) ::= sort_specification(B). { A = createNodeList(pCxt, B); } -sort_specification_list(A) ::= - sort_specification_list(B) NK_COMMA sort_specification(C). { A = addNodeToList(pCxt, B, C); } - -sort_specification(A) ::= - expression(B) ordering_specification_opt(C) null_ordering_opt(D). { A = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, B), C, D); } - -%type ordering_specification_opt EOrder -%destructor ordering_specification_opt { } -ordering_specification_opt(A) ::= . { A = ORDER_ASC; } -ordering_specification_opt(A) ::= ASC. { A = ORDER_ASC; } -ordering_specification_opt(A) ::= DESC. { A = ORDER_DESC; } - -%type null_ordering_opt ENullOrder -%destructor null_ordering_opt { } -null_ordering_opt(A) ::= . { A = NULL_ORDER_DEFAULT; } -null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; } -null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/parAst.h similarity index 99% rename from source/libs/parser/inc/astCreateFuncs.h rename to source/libs/parser/inc/parAst.h index bc43b0f4e9..11c56ddf3c 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/parAst.h @@ -22,9 +22,9 @@ extern "C" { #include "cmdnodes.h" #include "parser.h" -#include "parserUtil.h" +#include "parToken.h" +#include "parUtil.h" #include "querynodes.h" -#include "ttoken.h" typedef struct SAstCreateContext { SParseContext* pQueryCxt; diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/parInsertData.h similarity index 100% rename from source/libs/parser/inc/dataBlockMgt.h rename to source/libs/parser/inc/parInsertData.h diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parInt.h similarity index 93% rename from source/libs/parser/inc/parserInt.h rename to source/libs/parser/inc/parInt.h index 3fb3bbcc7f..af0d78717e 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parInt.h @@ -22,6 +22,7 @@ extern "C" { #include "parser.h" +int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery); int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); diff --git a/source/libs/parser/inc/ttoken.h b/source/libs/parser/inc/parToken.h similarity index 100% rename from source/libs/parser/inc/ttoken.h rename to source/libs/parser/inc/parToken.h diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parUtil.h similarity index 98% rename from source/libs/parser/inc/parserUtil.h rename to source/libs/parser/inc/parUtil.h index 50c99bb08b..4d7a8e2a18 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -22,7 +22,6 @@ extern "C" { #include "os.h" #include "query.h" -#include "ttoken.h" typedef struct SMsgBuf { int32_t len; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index fa713d5fb4..29e99c360a 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1,24 +1,12 @@ //lemon parser file to generate sql parse by using finite-state-machine code used to parse sql //usage: lemon sql.y + %token_prefix TK_ +%token_type { SToken } +%default_type { SNode* } +%default_destructor { nodesDestroyNode($$); } -%token_type {SToken} -%default_type {SToken} -%extra_argument {SSqlInfo* pInfo} - -%fallback ID BOOL INTEGER FLOAT STRING TIMESTAMP. - -%left OR. -%left AND. -%right NOT. -%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -%left GT GE LT LE. -%left BITAND BITOR LSHIFT RSHIFT. -%left PLUS MINUS. -%left DIVIDE TIMES. -%left STAR SLASH REM. -%left CONCAT. -%right UMINUS UPLUS BITNOT. +%extra_argument { SAstCreateContext* pCxt } %include { #include @@ -26,920 +14,542 @@ #include #include #include -#include "astGenerator.h" -#include "tmsgtype.h" -#include "ttoken.h" + +#include "nodes.h" +#include "parToken.h" #include "ttokendef.h" -#include "tvariant.h" -#include "parserInt.h" +#include "parAst.h" } -%syntax_error { - pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->msg); - int32_t len = 0; - +%syntax_error { if(TOKEN.z) { - char msg[] = "syntax error near \"%s\""; - int32_t sqlLen = strlen(&TOKEN.z[0]); - - if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { - char tmpstr[128] = {0}; - memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->msg, msg, tmpstr); - } else { - len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); - } - + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } else { - len = sprintf(pInfo->msg, "Incomplete SQL statement"); + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } - - assert(len <= outputBufLen); + pCxt->valid = false; } -%parse_accept {} - -program ::= cmd. {} - -//////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// -cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} -cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} -cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} -cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} -cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} -cmd ::= SHOW USERS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} - -cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } -cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } -cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} -cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } -cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TRANS, 0, 0); } -cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } - -cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } -cmd ::= SHOW VNODES ids(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } - - -%type dbPrefix {SToken} -dbPrefix(A) ::=. {A.n = 0; A.type = 0;} -dbPrefix(A) ::= ids(X) DOT. {A = X; } - -%type cpxName {SToken} -cpxName(A) ::= . {A.n = 0; } -cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } - -cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X); -} -cmd ::= SHOW CREATE STABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &X); -} - -cmd ::= SHOW CREATE DATABASE ids(X). { - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X); -} - -cmd ::= SHOW dbPrefix(X) TABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, &Y); -} - -cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &Y); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS. { - SToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); -} - -//drop configure for tables -cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1); -} - -//drop stable -cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE); -} - -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); } -cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); } -cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); } - -cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } -cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } -cmd ::= DROP ACCOUNT ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } - -/////////////////////////////////THE USE STATEMENT////////////////////////////////////////// -cmd ::= USE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &X);} - -/////////////////////////////////THE DESCRIBE STATEMENT///////////////////////////////////// -cmd ::= DESCRIBE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -cmd ::= DESC ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -/////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} -cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } -cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } -cmd ::= ALTER LOCAL ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } -cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -//cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} - -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} - -////////////////////////////// COMPACT STATEMENT ////////////////////////////////////////////// - -cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, Y);} - -// An IDENTIFIER can be a generic identifier, or one of several keywords. -// Any non-standard keyword can also be an identifier. -// And "ids" is an identifer-or-string. -%type ids {SToken} -ids(A) ::= ID(X). {A = X; } -//ids(A) ::= STRING(X). {A = X; } - -%type ifexists {SToken} -ifexists(X) ::= IF EXISTS. { X.n = 1;} -ifexists(X) ::= . { X.n = 0;} - -%type ifnotexists {SToken} -ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;} -ifnotexists(X) ::= . { X.n = 0;} - -/////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// -//create option for dnode/db/user/account -cmd ::= CREATE DNODE ids(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);} -cmd ::= CREATE DNODE IPTOKEN(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);} -cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} -cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -//cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);} -cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} - -bufsize(Y) ::= . { Y.n = 0; } -bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; } - -pps(Y) ::= . { Y.n = 0; } -pps(Y) ::= PPS INTEGER(X). { Y = X; } - -tseries(Y) ::= . { Y.n = 0; } -tseries(Y) ::= TSERIES INTEGER(X). { Y = X; } - -dbs(Y) ::= . { Y.n = 0; } -dbs(Y) ::= DBS INTEGER(X). { Y = X; } - -streams(Y) ::= . { Y.n = 0; } -streams(Y) ::= STREAMS INTEGER(X). { Y = X; } - -storage(Y) ::= . { Y.n = 0; } -storage(Y) ::= STORAGE INTEGER(X). { Y = X; } - -qtime(Y) ::= . { Y.n = 0; } -qtime(Y) ::= QTIME INTEGER(X). { Y = X; } - -users(Y) ::= . { Y.n = 0; } -users(Y) ::= USERS INTEGER(X). { Y = X; } - -conns(Y) ::= . { Y.n = 0; } -conns(Y) ::= CONNS INTEGER(X). { Y = X; } - -state(Y) ::= . { Y.n = 0; } -state(Y) ::= STATE ids(X). { Y = X; } - -%type acct_optr {SCreateAcctInfo} -acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { - Y.maxUsers = (K.n>0)?atoi(K.z):-1; - Y.maxDbs = (E.n>0)?atoi(E.z):-1; - Y.maxTimeSeries = (D.n>0)?atoi(D.z):-1; - Y.maxStreams = (F.n>0)?atoi(F.z):-1; - Y.maxPointsPerSecond = (C.n>0)?atoi(C.z):-1; - Y.maxStorage = (P.n>0)?strtoll(P.z, NULL, 10):-1; - Y.maxQueryTime = (Q.n>0)?strtoll(Q.z, NULL, 10):-1; - Y.maxConnections = (L.n>0)?atoi(L.z):-1; - Y.stat = M; -} - -%type intitemlist {SArray*} -%destructor intitemlist {taosArrayDestroy($$);} - -%type intitem {SVariant} -intitemlist(A) ::= intitemlist(X) COMMA intitem(Y). { A = tListItemAppend(X, &Y, -1); } -intitemlist(A) ::= intitem(X). { A = tListItemAppend(NULL, &X, -1); } - -intitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } - -%type keep {SArray*} -%destructor keep {taosArrayDestroy($$);} -keep(Y) ::= KEEP intitemlist(X). { Y = X; } - -cache(Y) ::= CACHE INTEGER(X). { Y = X; } -replica(Y) ::= REPLICA INTEGER(X). { Y = X; } -quorum(Y) ::= QUORUM INTEGER(X). { Y = X; } -days(Y) ::= DAYS INTEGER(X). { Y = X; } -minrows(Y) ::= MINROWS INTEGER(X). { Y = X; } -maxrows(Y) ::= MAXROWS INTEGER(X). { Y = X; } -blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; } -ctime(Y) ::= CTIME INTEGER(X). { Y = X; } -wal(Y) ::= WAL INTEGER(X). { Y = X; } -fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } -comp(Y) ::= COMP INTEGER(X). { Y = X; } -prec(Y) ::= PRECISION STRING(X). { Y = X; } -update(Y) ::= UPDATE INTEGER(X). { Y = X; } -cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } -vgroups(Y) ::= VGROUPS INTEGER(X). { Y = X; } -//partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; } -stream_mode(Y) ::= STREAM MODE INTEGER(X). { Y = X; } - -%type db_optr {SCreateDbInfo} -db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} - -db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) days(X). { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) minrows(X). { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) maxrows(X). { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } -db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) vgroups(X). { Y = Z; Y.numOfVgroups = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) stream_mode(X). { Y = Z; Y.streamMode = strtol(X.z, NULL, 10); } - -//%type topic_optr {SCreateDbInfo} -// -//topic_optr(Y) ::= db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -//topic_optr(Y) ::= topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type alter_db_optr {SCreateDbInfo} -alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} - -alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } - -// dynamically update the following two parameters are not allowed. -//alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -//alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } not support yet - -//%type alter_topic_optr {SCreateDbInfo} - -//alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -//alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type typename {SField} -typename(A) ::= ids(X). { - X.type = 0; - tSetColumnType (&A, &X); -} - -//define binary type, e.g., binary(10), nchar(10) -typename(A) ::= ids(X) LP signed(Y) RP. { - if (Y <= 0) { - X.type = 0; - tSetColumnType(&A, &X); - } else { - X.type = -Y; // negative value of name length - tSetColumnType(&A, &X); - } -} - -// define the unsigned number type -typename(A) ::= ids(X) UNSIGNED(Z). { - X.type = 0; - X.n = ((Z.z + Z.n) - X.z); - tSetColumnType (&A, &X); -} - -%type signed {int64_t} -signed(A) ::= INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} - -////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE create_table_args. {} -cmd ::= CREATE TABLE create_stable_args. {} -cmd ::= CREATE STABLE create_stable_args. {} -cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} - -%type create_table_list{SCreateTableSql*} -%destructor create_table_list{destroyCreateTableSql($$);} -create_table_list(A) ::= create_from_stable(Z). { - SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); - pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - - taosArrayPush(pCreateTable->childTableInfo, &Z); - pCreateTable->type = TSDB_SQL_CREATE_TABLE; - A = pCreateTable; -} - -create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { - taosArrayPush(X->childTableInfo, &Z); - A = X; -} - -%type create_table_args{SCreateTableSql*} -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { - A = tSetCreateTableInfo(X, NULL, NULL, TSDB_SQL_CREATE_TABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create super table -%type create_stable_args{SCreateTableSql*} -create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateTableInfo(X, Y, NULL, TSDB_SQL_CREATE_STABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_STABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create table by using super table -// create table table_name using super_table_name tags(tag_values1, tag_values2) -%type create_from_stable{SCreatedTableInfo} -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist1(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, NULL, Y, &V, &U); -} - -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist1(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, P, Y, &V, &U); -} - -%type tagNamelist{SArray*} -%destructor tagNamelist {taosArrayDestroy($$);} -tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; } -tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SToken)); taosArrayPush(A, &X);} - -// create stream -// create table table_name as select count(*) from super_table_name interval(time) -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { -// A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM); -// setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); -// -// V.n += Z.n; -// setCreatedTableName(pInfo, &V, &U); -} - -%type column{SField} -%type columnlist{SArray*} -%destructor columnlist {taosArrayDestroy($$);} -columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; } -columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);} - -// The information used for a column is the name and type of column: -// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) -column(A) ::= ids(X) typename(Y). { - tSetColumnInfo(&A, &X, &Y); -} - -%type tagitemlist1 {SArray*} -%destructor tagitemlist1 {taosArrayDestroy($$);} - -tagitemlist1(A) ::= tagitemlist1(X) COMMA tagitem1(Y). { taosArrayPush(X, &Y); A = X;} -tagitemlist1(A) ::= tagitem1(Y). { A = taosArrayInit(4, sizeof(SToken)); taosArrayPush(A, &Y); } - -%type tagitem1 {SToken} -tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= INTEGER(X). { A = X; } -tagitem1(A) ::= FLOAT(X). { A = X; } -tagitem1(A) ::= STRING(X). { A = X; } -tagitem1(A) ::= BOOL(X). { A = X; } -tagitem1(A) ::= NULL(X). { A = X; } -tagitem1(A) ::= NOW(X). { A = X; } - -%type tagitemlist {SArray*} -%destructor tagitemlist {taosArrayDestroy($$);} - -%type tagitem {SVariant} -tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tListItemAppend(X, &Y, -1); } -tagitemlist(A) ::= tagitem(X). { A = tListItemAppend(NULL, &X, -1); } - -tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= STRING(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= BOOL(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= NULL(X). { X.type = 0; taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&A, X.z, X.n, X.type);} - -tagitem(A) ::= MINUS(X) INTEGER(Y).{ - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -tagitem(A) ::= MINUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -tagitem(A) ::= PLUS(X) INTEGER(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -tagitem(A) ::= PLUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -//////////////////////// The SELECT statement ///////////////////////////////// -%type select {SSqlNode*} -%destructor select {destroySqlNode($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_option(K) sliding_opt(S) session_option(H) windowstate_option(D) fill_opt(F)groupby_opt(P) having_opt(N) orderby_opt(Z) slimit_opt(G) limit_opt(L). { - A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &D, &S, F, &L, &G, N); -} - -select(A) ::= LP select(B) RP. {A = B;} - -%type union {SSubclause*} -%destructor union {destroyAllSqlNode($$);} -union(Y) ::= select(X). { Y = setSubclause(NULL, X); } -union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, SQL_TYPE_UNIONALL, X); } -union(Y) ::= union(Z) UNION select(X). { Y = appendSelectClause(Z, SQL_TYPE_UNION, X); } -cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } - -// Support for the SQL exprssion without from & where subclauses, e.g., -// select database() -// select server_version() -// select client_version() -// select server_state() -select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -} - -// selcollist is a list of expressions that are to become the return -// values of the SELECT statement. The "*" in statements like -// "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. -%type selcollist {SArray*} -%destructor selcollist {tSqlExprListDestroy($$);} - -%type sclp {SArray*} -%destructor sclp {tSqlExprListDestroy($$);} -sclp(A) ::= selcollist(X) COMMA. {A = X;} -sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { - A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0); -} - -selcollist(A) ::= sclp(P) STAR. { - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - A = tSqlExprListAppend(P, pNode, 0, 0); -} - -// An option "AS " phrase that can follow one of the expressions that -// define the result set, or one of the tables in the FROM clause. -%type as {SToken} -as(X) ::= AS ids(Y). { X = Y; } -as(X) ::= ids(Y). { X = Y; } -as(X) ::= . { X.n = 0; } - -%type distinct {SToken} -distinct(X) ::= DISTINCT(Y). { X = Y; } -distinct(X) ::= . { X.n = 0;} - -// A complete FROM clause. -%type from {SRelationInfo*} -%destructor from {destroyRelationInfo($$);} -from(A) ::= FROM tablelist(X). {A = X;} -from(A) ::= FROM sub(X). {A = X;} - -%type sub {SRelationInfo*} -%destructor sub {destroyRelationInfo($$);} -sub(A) ::= LP union(Y) RP. {A = addSubquery(NULL, Y, NULL);} -sub(A) ::= LP union(Y) RP ids(Z). {A = addSubquery(NULL, Y, &Z);} -sub(A) ::= sub(X) COMMA LP union(Y) RP ids(Z).{A = addSubquery(X, Y, &Z);} - -%type tablelist {SRelationInfo*} -%destructor tablelist {destroyRelationInfo($$);} -tablelist(A) ::= ids(X) cpxName(Y). { - X.n += Y.n; - A = setTableNameList(NULL, &X, NULL); -} - -tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { - X.n += Y.n; - A = setTableNameList(NULL, &X, &Z); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { - X.n += Z.n; - A = setTableNameList(Y, &X, NULL); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { - X.n += Z.n; - A = setTableNameList(Y, &X, &F); -} - -// The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SToken} -tmvar(A) ::= VARIABLE(X). {A = X;} - -%type interval_option {SIntervalVal} -interval_option(N) ::= intervalKey(A) LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.token = A;} -interval_option(N) ::= intervalKey(A) LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X; N.token = A;} -interval_option(N) ::= . {memset(&N, 0, sizeof(N));} - -%type intervalKey {int32_t} -intervalKey(A) ::= INTERVAL. {A = TK_INTERVAL;} -intervalKey(A) ::= EVERY. {A = TK_EVERY; } - -%type session_option {SSessionWindowVal} -session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;} -session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. { - V.n += Z.n; - X.col = V; - X.gap = Y; -} - -%type windowstate_option {SWindowStateVal} -windowstate_option(X) ::= . { X.col.n = 0; X.col.z = NULL;} -windowstate_option(X) ::= STATE_WINDOW LP ids(V) RP. { X.col = V; } - -%type fill_opt {SArray*} -%destructor fill_opt {taosArrayDestroy($$);} -fill_opt(N) ::= . { N = 0; } -fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. { - SVariant A = {0}; - toTSDBType(Y.type); - taosVariantCreate(&A, Y.z, Y.n, Y.type); - - tListItemInsert(X, &A, -1, 0); - N = X; -} - -fill_opt(N) ::= FILL LP ID(Y) RP. { - toTSDBType(Y.type); - N = tListItemAppendToken(NULL, &Y, -1); -} - -%type sliding_opt {SToken} -sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } -sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } - -%type orderby_opt {SArray*} -%destructor orderby_opt {taosArrayDestroy($$);} - -%type sortlist {SArray*} -%destructor sortlist {taosArrayDestroy($$);} - -%type sortitem {SVariant} -%destructor sortitem {taosVariantDestroy(&$$);} - -orderby_opt(A) ::= . {A = 0;} -orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} - -sortlist(A) ::= sortlist(X) COMMA item(Y) sortorder(Z). { - A = tListItemAppend(X, &Y, Z); -} - -sortlist(A) ::= item(Y) sortorder(Z). { - A = tListItemAppend(NULL, &Y, Z); -} - -%type item {SVariant} -item(A) ::= ids(X) cpxName(Y). { - toTSDBType(X.type); - X.n += Y.n; - - taosVariantCreate(&A, X.z, X.n, X.type); -} - -%type sortorder {int} -sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; } -sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;} -sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default - -//group by clause -%type groupby_opt {SArray*} -%destructor groupby_opt {taosArrayDestroy($$);} -%type grouplist {SArray*} -%destructor grouplist {taosArrayDestroy($$);} - -groupby_opt(A) ::= . { A = 0;} -groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;} - -grouplist(A) ::= grouplist(X) COMMA item(Y). { - A = tListItemAppend(X, &Y, -1); -} - -grouplist(A) ::= item(X). { - A = tListItemAppend(NULL, &X, -1); -} - -//having clause, ignore the input condition in having -%type having_opt {tSqlExpr*} -%destructor having_opt {tSqlExprDestroy($$);} -having_opt(A) ::=. {A = 0;} -having_opt(A) ::= HAVING expr(X). {A = X;} - -//limit-offset subclause -%type limit_opt {SLimit} -limit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y). - { A.limit = X; A.offset = Y;} -limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y). - { A.limit = Y; A.offset = X;} - -%type slimit_opt {SLimit} -slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X). {A.limit = X; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X) SOFFSET signed(Y). - {A.limit = X; A.offset = Y;} -slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). - {A.limit = Y; A.offset = X;} - -%type where_opt {tSqlExpr*} -%destructor where_opt {tSqlExprDestroy($$);} - -where_opt(A) ::= . {A = 0;} -where_opt(A) ::= WHERE expr(X). {A = X;} - -/////////////////////////// Expression Processing ///////////////////////////// -// -%type expr {tSqlExpr*} -%destructor expr {tSqlExprDestroy($$);} - -expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->exprToken.z = X.z; A->exprToken.n = (Z.z - X.z + 1);} - -expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} -expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} - -// ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); } - -// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } - -// is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} - -// relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} - -expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSqlExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);} - -expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } - -// binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } - -// like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } - -// match expression -expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); } -expr(A) ::= expr(X) NMATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_NMATCH); } - -//in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); } - -%type exprlist {SArray*} -%destructor exprlist {tSqlExprListDestroy($$);} - -%type expritem {tSqlExpr*} -%destructor expritem {tSqlExprDestroy($$);} - -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);} -exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);} -expritem(A) ::= expr(X). {A = X;} -expritem(A) ::= . {A = 0;} - -///////////////////////////////////reset query cache////////////////////////////////////// -cmd ::= RESET QUERY CACHE. { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} - -///////////////////////////////////sync replica database////////////////////////////////// -cmd ::= SYNCDB ids(X) REPLICA.{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &X);} - -///////////////////////////////////ALTER TABLE statement////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - toTSDBType(A.type); - SArray* K = tListItemAppendToken(NULL, &A, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tListItemAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - A = tListItemAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -///////////////////////////////////ALTER STABLE statement////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - - toTSDBType(A.type); - SArray* K = tListItemAppendToken(NULL, &A, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tListItemAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - A = tListItemAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} - -%fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED - DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD - LIKE MATCH NMATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES. +%left OR. +%left AND. +//%right NOT. +%left UNION ALL MINUS EXCEPT INTERSECT. +%left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. +%left NK_PLUS NK_MINUS. +//%left DIVIDE TIMES. +%left NK_STAR NK_SLASH NK_REM. +%left NK_CONCAT. +//%right NK_BITNOT. + +/************************************************ create/alter/drop/show user *****************************************/ +cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B);} +cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B);} +cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B);} +cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } +cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } + +/************************************************ create/drop/show dnode **********************************************/ +cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL);} +cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B);} +cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} +cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} +cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } + +%type dnode_endpoint { SToken } +%destructor dnode_endpoint { } +dnode_endpoint(A) ::= NK_STRING(B). { A = B; } + +%type dnode_host_name { SToken } +%destructor dnode_host_name { } +dnode_host_name(A) ::= NK_ID(B). { A = B; } +dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } + +/************************************************ create/drop/show/use database ***************************************/ +cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C);} +cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } +cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } +cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A);} + +%type not_exists_opt { bool } +%destructor not_exists_opt { } +not_exists_opt(A) ::= IF NOT EXISTS. { A = true; } +not_exists_opt(A) ::= . { A = false; } + +%type exists_opt { bool } +%destructor exists_opt { } +exists_opt(A) ::= IF EXISTS. { A = true; } +exists_opt(A) ::= . { A = false; } + +%type db_options { SDatabaseOptions* } +%destructor db_options { tfree($$); } +db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } +db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BLOCKS, &C); } +db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHE, &C); } +db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } +db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } +db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } +db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); } +db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); } +db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); } +db_options(A) ::= db_options(B) KEEP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, &C); } +db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } +db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); } +db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } +db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } +db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } +db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } +db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } +db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } + +/************************************************ create/drop/show table/stable ***************************************/ +cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) + NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} +cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} +cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) + NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} +cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } +cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } +cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } +cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } + +%type multi_create_clause { SNodeList* } +%destructor multi_create_clause { nodesDestroyList($$); } +multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } +multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); } + +create_subtable_clause(A) ::= + not_exists_opt(B) full_table_name(C) USING full_table_name(D) + specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP. { A = createCreateSubTableClause(pCxt, B, C, D, E, F); } + +%type multi_drop_clause { SNodeList* } +%destructor multi_drop_clause { nodesDestroyList($$); } +multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); } +multi_drop_clause(A) ::= multi_drop_clause(B) drop_table_clause(C). { A = addNodeToList(pCxt, B, C); } + +drop_table_clause(A) ::= exists_opt(B) full_table_name(C). { A = createDropTableClause(pCxt, B, C); } + +%type specific_tags_opt { SNodeList* } +%destructor specific_tags_opt { nodesDestroyList($$); } +specific_tags_opt(A) ::= . { A = NULL; } +specific_tags_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } + +full_table_name(A) ::= table_name(B). { A = createRealTableNode(pCxt, NULL, &B, NULL); } +full_table_name(A) ::= db_name(B) NK_DOT table_name(C). { A = createRealTableNode(pCxt, &B, &C, NULL); } + +%type column_def_list { SNodeList* } +%destructor column_def_list { nodesDestroyList($$); } +column_def_list(A) ::= column_def(B). { A = createNodeList(pCxt, B); } +column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } + +column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } +column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } + +%type type_name { SDataType } +%destructor type_name { } +type_name(A) ::= BOOL. { A = createDataType(TSDB_DATA_TYPE_BOOL); } +type_name(A) ::= TINYINT. { A = createDataType(TSDB_DATA_TYPE_TINYINT); } +type_name(A) ::= SMALLINT. { A = createDataType(TSDB_DATA_TYPE_SMALLINT); } +type_name(A) ::= INT. { A = createDataType(TSDB_DATA_TYPE_INT); } +type_name(A) ::= INTEGER. { A = createDataType(TSDB_DATA_TYPE_INT); } +type_name(A) ::= BIGINT. { A = createDataType(TSDB_DATA_TYPE_BIGINT); } +type_name(A) ::= FLOAT. { A = createDataType(TSDB_DATA_TYPE_FLOAT); } +type_name(A) ::= DOUBLE. { A = createDataType(TSDB_DATA_TYPE_DOUBLE); } +type_name(A) ::= BINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &B); } +type_name(A) ::= TIMESTAMP. { A = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +type_name(A) ::= NCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &B); } +type_name(A) ::= TINYINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UTINYINT); } +type_name(A) ::= SMALLINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_USMALLINT); } +type_name(A) ::= INT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UINT); } +type_name(A) ::= BIGINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UBIGINT); } +type_name(A) ::= JSON. { A = createDataType(TSDB_DATA_TYPE_JSON); } +type_name(A) ::= VARCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &B); } +type_name(A) ::= MEDIUMBLOB. { A = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } +type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } +type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } + +%type tags_def_opt { SNodeList* } +%destructor tags_def_opt { nodesDestroyList($$); } +tags_def_opt(A) ::= . { A = NULL; } +tags_def_opt(A) ::= tags_def(B). { A = B; } + +%type tags_def { SNodeList* } +%destructor tags_def { nodesDestroyList($$); } +tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } + +%type table_options { STableOptions* } +%destructor table_options { tfree($$); } +table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} +table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } +table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } +table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } +table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } + +%type col_name_list { SNodeList* } +%destructor col_name_list { nodesDestroyList($$); } +col_name_list(A) ::= col_name(B). { A = createNodeList(pCxt, B); } +col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } + +col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } + +/************************************************ show vgroups ********************************************************/ +cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } +cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); } + +/************************************************ select **************************************************************/ +cmd ::= query_expression(A). { pCxt->pRootNode = A; } + +/************************************************ literal *************************************************************/ +literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } +literal(A) ::= NK_FLOAT(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); } +literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)); } +literal(A) ::= NK_BOOL(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); } +literal(A) ::= TIMESTAMP(B) NK_STRING(C). { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); } +literal(A) ::= duration_literal(B). { A = B; } + +duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } + +%type literal_list { SNodeList* } +%destructor literal_list { nodesDestroyList($$); } +literal_list(A) ::= literal(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } + +/************************************************ names and identifiers ***********************************************/ +%type db_name { SToken } +%destructor db_name { } +db_name(A) ::= NK_ID(B). { A = B; } + +%type table_name { SToken } +%destructor table_name { } +table_name(A) ::= NK_ID(B). { A = B; } + +%type column_name { SToken } +%destructor column_name { } +column_name(A) ::= NK_ID(B). { A = B; } + +%type function_name { SToken } +%destructor function_name { } +function_name(A) ::= NK_ID(B). { A = B; } + +%type table_alias { SToken } +%destructor table_alias { } +table_alias(A) ::= NK_ID(B). { A = B; } + +%type column_alias { SToken } +%destructor column_alias { } +column_alias(A) ::= NK_ID(B). { A = B; } + +%type user_name { SToken } +%destructor user_name { } +user_name(A) ::= NK_ID(B). { A = B; } + +/************************************************ expression **********************************************************/ +expression(A) ::= literal(B). { A = B; } +//expression(A) ::= NK_QUESTION(B). { A = B; } +//expression(A) ::= pseudo_column(B). { A = B; } +expression(A) ::= column_reference(B). { A = B; } +expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } +expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); } +//expression(A) ::= cast_expression(B). { A = B; } +//expression(A) ::= case_expression(B). { A = B; } +expression(A) ::= subquery(B). { A = B; } +expression(A) ::= NK_LP(B) expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, releaseRawExprNode(pCxt, C)); } +expression(A) ::= NK_PLUS(B) expression(C). { + SToken t = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &B, &t, releaseRawExprNode(pCxt, C)); + } +expression(A) ::= NK_MINUS(B) expression(C). { + SToken t = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL)); + } +expression(A) ::= expression(B) NK_PLUS expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_MINUS expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_STAR expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_SLASH expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_REM expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } + +%type expression_list { SNodeList* } +%destructor expression_list { nodesDestroyList($$); } +expression_list(A) ::= expression(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } + +column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } +column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } + +//pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); } + +/************************************************ predicate ***********************************************************/ +predicate(A) ::= expression(B) compare_op(C) expression(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } +//predicate(A) ::= expression(B) compare_op sub_type expression(B). +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); + } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } +predicate(A) ::= expression(B) IS NULL(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, B), NULL)); + } +predicate(A) ::= expression(B) IS NOT NULL(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, B), NULL)); + } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } + +%type compare_op { EOperatorType } +%destructor compare_op { } +compare_op(A) ::= NK_LT. { A = OP_TYPE_LOWER_THAN; } +compare_op(A) ::= NK_GT. { A = OP_TYPE_GREATER_THAN; } +compare_op(A) ::= NK_LE. { A = OP_TYPE_LOWER_EQUAL; } +compare_op(A) ::= NK_GE. { A = OP_TYPE_GREATER_EQUAL; } +compare_op(A) ::= NK_NE. { A = OP_TYPE_NOT_EQUAL; } +compare_op(A) ::= NK_EQ. { A = OP_TYPE_EQUAL; } +compare_op(A) ::= LIKE. { A = OP_TYPE_LIKE; } +compare_op(A) ::= NOT LIKE. { A = OP_TYPE_NOT_LIKE; } +compare_op(A) ::= MATCH. { A = OP_TYPE_MATCH; } +compare_op(A) ::= NMATCH. { A = OP_TYPE_NMATCH; } + +%type in_op { EOperatorType } +%destructor in_op { } +in_op(A) ::= IN. { A = OP_TYPE_IN; } +in_op(A) ::= NOT IN. { A = OP_TYPE_NOT_IN; } + +in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } + +/************************************************ boolean_value_expression ********************************************/ +boolean_value_expression(A) ::= boolean_primary(B). { A = B; } +boolean_value_expression(A) ::= NOT(C) boolean_primary(B). { + SToken e = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); + } +boolean_value_expression(A) ::= + boolean_value_expression(B) OR boolean_value_expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +boolean_value_expression(A) ::= + boolean_value_expression(B) AND boolean_value_expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } + +boolean_primary(A) ::= predicate(B). { A = B; } +boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, releaseRawExprNode(pCxt, B)); } + +/************************************************ common_expression ********************************************/ +common_expression(A) ::= expression(B). { A = B; } +common_expression(A) ::= boolean_value_expression(B). { A = B; } + +/************************************************ from_clause *********************************************************/ +from_clause(A) ::= FROM table_reference_list(B). { A = B; } + +table_reference_list(A) ::= table_reference(B). { A = B; } +table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } + +/************************************************ table_reference *****************************************************/ +table_reference(A) ::= table_primary(B). { A = B; } +table_reference(A) ::= joined_table(B). { A = B; } + +table_primary(A) ::= table_name(B) alias_opt(C). { A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) alias_opt(C). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), &C); } +table_primary(A) ::= parenthesized_joined_table(B). { A = B; } + +%type alias_opt { SToken } +%destructor alias_opt { } +alias_opt(A) ::= . { A = nil_token; } +alias_opt(A) ::= table_alias(B). { A = B; } +alias_opt(A) ::= AS table_alias(B). { A = B; } + +parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { A = B; } +parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { A = B; } + +/************************************************ joined_table ********************************************************/ +joined_table(A) ::= + table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { A = createJoinTableNode(pCxt, C, B, D, E); } + +%type join_type { EJoinType } +%destructor join_type { } +join_type(A) ::= . { A = JOIN_TYPE_INNER; } +join_type(A) ::= INNER. { A = JOIN_TYPE_INNER; } + +/************************************************ query_specification *************************************************/ +query_specification(A) ::= + SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) + partition_by_clause_opt(F) twindow_clause_opt(G) + group_by_clause_opt(H) having_clause_opt(I). { + A = createSelectStmt(pCxt, B, C, D); + A = addWhereClause(pCxt, A, E); + A = addPartitionByClause(pCxt, A, F); + A = addWindowClauseClause(pCxt, A, G); + A = addGroupByClause(pCxt, A, H); + A = addHavingClause(pCxt, A, I); + } + +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt { } +set_quantifier_opt(A) ::= . { A = false; } +set_quantifier_opt(A) ::= DISTINCT. { A = true; } +set_quantifier_opt(A) ::= ALL. { A = false; } + +%type select_list { SNodeList* } +%destructor select_list { nodesDestroyList($$); } +select_list(A) ::= NK_STAR. { A = NULL; } +select_list(A) ::= select_sublist(B). { A = B; } + +%type select_sublist { SNodeList* } +%destructor select_sublist { nodesDestroyList($$); } +select_sublist(A) ::= select_item(B). { A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { A = addNodeToList(pCxt, B, C); } + +select_item(A) ::= common_expression(B). { + SToken t = getTokenFromRawExprNode(pCxt, B); + A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t); + } +select_item(A) ::= common_expression(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= common_expression(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { A = createColumnNode(pCxt, &B, &C); } + +where_clause_opt(A) ::= . { A = NULL; } +where_clause_opt(A) ::= WHERE search_condition(B). { A = B; } + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { nodesDestroyList($$); } +partition_by_clause_opt(A) ::= . { A = NULL; } +partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { A = B; } + +twindow_clause_opt(A) ::= . { A = NULL; } +twindow_clause_opt(A) ::= + SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, B, NULL, C, D); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, B, C, D, E); } + +sliding_opt(A) ::= . { A = NULL; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = B; } + +fill_opt(A) ::= . { A = NULL; } +fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } + +%type fill_mode { EFillMode } +%destructor fill_mode { } +fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; } +fill_mode(A) ::= PREV. { A = FILL_MODE_PREV; } +fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; } +fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; } +fill_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; } + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { nodesDestroyList($$); } +group_by_clause_opt(A) ::= . { A = NULL; } +group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { A = B; } + +%type group_by_list { SNodeList* } +%destructor group_by_list { nodesDestroyList($$); } +group_by_list(A) ::= expression(B). { A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); } +group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); } + +having_clause_opt(A) ::= . { A = NULL; } +having_clause_opt(A) ::= HAVING search_condition(B). { A = B; } + +/************************************************ query_expression ****************************************************/ +query_expression(A) ::= + query_expression_body(B) + order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + A = addOrderByClause(pCxt, B, C); + A = addSlimitClause(pCxt, A, D); + A = addLimitClause(pCxt, A, E); + } + +query_expression_body(A) ::= query_primary(B). { A = B; } +query_expression_body(A) ::= + query_expression_body(B) UNION ALL query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { A = B; } +//query_primary(A) ::= +// NK_LP query_expression_body(B) +// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { nodesDestroyList($$); } +order_by_clause_opt(A) ::= . { A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { A = B; } + +slimit_clause_opt(A) ::= . { A = NULL; } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } + +limit_clause_opt(A) ::= . { A = NULL; } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } + +/************************************************ subquery ************************************************************/ +subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, C); } + +/************************************************ search_condition ****************************************************/ +search_condition(A) ::= common_expression(B). { A = releaseRawExprNode(pCxt, B); } + +/************************************************ sort_specification_list *********************************************/ +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { nodesDestroyList($$); } +sort_specification_list(A) ::= sort_specification(B). { A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= + sort_specification_list(B) NK_COMMA sort_specification(C). { A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= + expression(B) ordering_specification_opt(C) null_ordering_opt(D). { A = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, B), C, D); } + +%type ordering_specification_opt EOrder +%destructor ordering_specification_opt { } +ordering_specification_opt(A) ::= . { A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { A = ORDER_DESC; } + +%type null_ordering_opt ENullOrder +%destructor null_ordering_opt { } +null_ordering_opt(A) ::= . { A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/parAstCreater.c similarity index 99% rename from source/libs/parser/src/astCreateFuncs.c rename to source/libs/parser/src/parAstCreater.c index 17a3d1f765..f26c4c3549 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/parAstCreater.c @@ -14,8 +14,8 @@ * along with this program. If not, see . */ -#include "astCreateFuncs.h" -#include "parserUtil.h" +#include "parAst.h" +#include "parUtil.h" #define CHECK_OUT_OF_MEM(p) \ do { \ diff --git a/source/libs/parser/src/astParse.c b/source/libs/parser/src/parAstParser.c similarity index 80% rename from source/libs/parser/src/astParse.c rename to source/libs/parser/src/parAstParser.c index 8be03b54b3..e992f150aa 100644 --- a/source/libs/parser/src/astParse.c +++ b/source/libs/parser/src/parAstParser.c @@ -13,28 +13,28 @@ * along with this program. If not, see . */ -#include "parserInt.h" +#include "parInt.h" -#include "astCreateFuncs.h" -#include "ttoken.h" +#include "parAst.h" +#include "parToken.h" typedef void* (*FMalloc)(size_t); typedef void (*FFree)(void*); -extern void* NewParseAlloc(FMalloc); -extern void NewParse(void*, int, SToken, void*); -extern void NewParseFree(void*, FFree); -extern void NewParseTrace(FILE*, char*); +extern void* ParseAlloc(FMalloc); +extern void Parse(void*, int, SToken, void*); +extern void ParseFree(void*, FFree); +extern void ParseTrace(FILE*, char*); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); - void *pParser = NewParseAlloc(malloc); + void *pParser = ParseAlloc(malloc); int32_t i = 0; while (1) { SToken t0 = {0}; if (cxt.pQueryCxt->pSql[i] == 0) { - NewParse(pParser, 0, t0, &cxt); + Parse(pParser, 0, t0, &cxt); goto abort_parse; } t0.n = tGetToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); @@ -47,7 +47,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { break; } case TK_NK_SEMI: { - NewParse(pParser, 0, t0, &cxt); + Parse(pParser, 0, t0, &cxt); goto abort_parse; } case TK_NK_QUESTION: @@ -64,8 +64,8 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { goto abort_parse; } default: - NewParse(pParser, t0.type, t0, &cxt); - // NewParseTrace(stdout, ""); + Parse(pParser, t0.type, t0, &cxt); + // ParseTrace(stdout, ""); if (!cxt.valid) { goto abort_parse; } @@ -73,7 +73,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { } abort_parse: - NewParseFree(pParser, free); + ParseFree(pParser, free); if (cxt.valid) { *pQuery = calloc(1, sizeof(SQuery)); if (NULL == *pQuery) { diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/parInsert.c similarity index 99% rename from source/libs/parser/src/insertParser.c rename to source/libs/parser/src/parInsert.c index 3241dc196b..43cc308483 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/parInsert.c @@ -13,13 +13,12 @@ * along with this program. If not, see . */ -#include "insertParser.h" - -#include "dataBlockMgt.h" -#include "parserUtil.h" +#include "parInsertData.h" +#include "parInt.h" +#include "parUtil.h" +#include "parToken.h" #include "tglobal.h" #include "ttime.h" -#include "ttoken.h" #include "ttypes.h" #define NEXT_TOKEN(pSql, sToken) \ diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/parInsertData.c similarity index 99% rename from source/libs/parser/src/dataBlockMgt.c rename to source/libs/parser/src/parInsertData.c index 34e440dd19..da5a652018 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/parInsertData.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "dataBlockMgt.h" +#include "parInsertData.h" #include "catalog.h" -#include "parserUtil.h" +#include "parUtil.h" #include "querynodes.h" #define IS_RAW_PAYLOAD(t) \ diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/parTokenizer.c similarity index 99% rename from source/libs/parser/src/ttokenizer.c rename to source/libs/parser/src/parTokenizer.c index b6435e95ad..089b9c30b5 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -14,10 +14,9 @@ */ #include "os.h" - +#include "parToken.h" #include "thash.h" #include "taosdef.h" -#include "ttoken.h" #include "ttokendef.h" // All the keywords of the SQL language are stored in a hash table diff --git a/source/libs/parser/src/astTranslate.c b/source/libs/parser/src/parTranslater.c similarity index 98% rename from source/libs/parser/src/astTranslate.c rename to source/libs/parser/src/parTranslater.c index 8c45fb2bff..0ed73ea1f6 100644 --- a/source/libs/parser/src/astTranslate.c +++ b/source/libs/parser/src/parTranslater.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ -#include "parserInt.h" +#include "parInt.h" #include "catalog.h" #include "cmdnodes.h" #include "functionMgt.h" -#include "parserUtil.h" +#include "parUtil.h" #include "ttime.h" static bool afterGroupBy(ESqlClause clause) { @@ -891,7 +891,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt if (TSDB_SUPER_TABLE == pTableMeta->tableType) { code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); } else { - // todo; + // todo : drop normal table or child table code = TSDB_CODE_FAILED; } } @@ -1106,7 +1106,7 @@ static int32_t translateShowTables(STranslateContext* pCxt) { if (NULL== pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } - pCxt->pCmdMsg->epSet = info->epset; + pCxt->pCmdMsg->epSet = info->epSet; pCxt->pCmdMsg->msgType = TDMT_VND_SHOW_TABLES; pCxt->pCmdMsg->msgLen = sizeof(SVShowTablesReq); pCxt->pCmdMsg->pMsg = pShowReq; @@ -1203,6 +1203,9 @@ static int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) { } static void destroyTranslateContext(STranslateContext* pCxt) { + if (NULL != pCxt->pNsLevel) { + + } taosArrayDestroy(pCxt->pNsLevel); if (NULL != pCxt->pCmdMsg) { tfree(pCxt->pCmdMsg->pMsg); @@ -1222,6 +1225,11 @@ static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema strcpy(pSchema->name, pCol->colName); } +static void destroyCreateTbReq(SVCreateTbReq* pReq) { + tfree(pReq->name); + tfree(pReq->ntbCfg.pSchema); +} + static int32_t buildNormalTableBatchReq( const char* pTableName, const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { SVCreateTbReq req = {0}; @@ -1230,6 +1238,7 @@ static int32_t buildNormalTableBatchReq( req.ntbCfg.nCols = LIST_LENGTH(pColumns); req.ntbCfg.pSchema = calloc(req.ntbCfg.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } SNode* pCol; @@ -1242,6 +1251,7 @@ static int32_t buildNormalTableBatchReq( pBatch->info = *pVgroupInfo; pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } taosArrayPush(pBatch->req.pArray, &req); @@ -1311,18 +1321,20 @@ static int32_t rewriteToVnodeModifOpStmt(SQuery* pQuery, SArray* pBufArray) { } static int32_t buildCreateTableDataBlock(const SCreateTableStmt* pStmt, const SVgroupInfo* pInfo, SArray** pBufArray) { + *pBufArray = taosArrayInit(1, POINTER_BYTES); + if (NULL == *pBufArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SVgroupTablesBatch tbatch = {0}; int32_t code = buildNormalTableBatchReq(pStmt->tableName, pStmt->pCols, pInfo, &tbatch); - if (TSDB_CODE_SUCCESS == code) { - *pBufArray = taosArrayInit(1, POINTER_BYTES); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } if (TSDB_CODE_SUCCESS == code) { code = serializeVgroupTablesBatch(&tbatch, *pBufArray); } destroyCreateTbReqBatch(&tbatch); + if (TSDB_CODE_SUCCESS != code) { + // todo : destroyCreateTbReqArray(*pBufArray); + } return code; } @@ -1331,7 +1343,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { SVgroupInfo info = {0}; int32_t code = getTableHashVgroup(pCxt->pParseCxt, pStmt->dbName, pStmt->tableName, &info); - SArray* pBufArray; + SArray* pBufArray = NULL; if (TSDB_CODE_SUCCESS == code) { code = buildCreateTableDataBlock(pStmt, &info, &pBufArray); } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parUtil.c similarity index 99% rename from source/libs/parser/src/parserUtil.c rename to source/libs/parser/src/parUtil.c index 73e62edc3c..aa2516e2b9 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "parserUtil.h" +#include "parUtil.h" static char* getSyntaxErrFormat(int32_t errCode) { switch (errCode) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 64418ad961..868bd75520 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -15,9 +15,8 @@ #include "parser.h" -#include "insertParser.h" -#include "parserInt.h" -#include "ttoken.h" +#include "parInt.h" +#include "parToken.h" static bool isInsertSql(const char* pStr, size_t length) { int32_t index = 0; @@ -47,5 +46,14 @@ int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { } void qDestroyQuery(SQuery* pQueryNode) { - // todo + if (NULL == pQueryNode) { + return; + } + nodesDestroyNode(pQueryNode->pRoot); + tfree(pQueryNode->pResSchema); + if (NULL != pQueryNode->pCmdMsg) { + tfree(pQueryNode->pCmdMsg->pMsg); + tfree(pQueryNode->pCmdMsg); + } + tfree(pQueryNode); } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/sql.c similarity index 97% rename from source/libs/parser/src/new_sql.c rename to source/libs/parser/src/sql.c index 058e1b35cd..34d38c1c06 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/sql.c @@ -33,9 +33,9 @@ #include #include "nodes.h" -#include "ttoken.h" +#include "parToken.h" #include "ttokendef.h" -#include "astCreateFuncs.h" +#include "parAst.h" /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -59,7 +59,7 @@ ** YYACTIONTYPE is the data type used for "action codes" - numbers ** that indicate what to do in response to the next ** token. -** NewParseTOKENTYPE is the data type used for minor type for terminal +** ParseTOKENTYPE is the data type used for minor type for terminal ** symbols. Background: A "minor type" is a semantic ** value associated with a terminal or non-terminal ** symbols. For example, for an "ID" terminal symbol, @@ -70,16 +70,16 @@ ** symbols. ** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of -** which is NewParseTOKENTYPE. The entry in the union +** which is ParseTOKENTYPE. The entry in the union ** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If ** zero the stack is dynamically sized using realloc() -** NewParseARG_SDECL A static variable declaration for the %extra_argument -** NewParseARG_PDECL A parameter declaration for the %extra_argument -** NewParseARG_PARAM Code to pass %extra_argument as a subroutine parameter -** NewParseARG_STORE Code to store %extra_argument into yypParser -** NewParseARG_FETCH Code to extract %extra_argument from yypParser -** NewParseCTX_* As NewParseARG_ except for %extra_context +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -101,10 +101,10 @@ #define YYCODETYPE unsigned char #define YYNOCODE 208 #define YYACTIONTYPE unsigned short int -#define NewParseTOKENTYPE SToken +#define ParseTOKENTYPE SToken typedef union { int yyinit; - NewParseTOKENTYPE yy0; + ParseTOKENTYPE yy0; ENullOrder yy9; SDatabaseOptions* yy103; SToken yy161; @@ -121,16 +121,16 @@ typedef union { #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif -#define NewParseARG_SDECL SAstCreateContext* pCxt ; -#define NewParseARG_PDECL , SAstCreateContext* pCxt -#define NewParseARG_PARAM ,pCxt -#define NewParseARG_FETCH SAstCreateContext* pCxt =yypParser->pCxt ; -#define NewParseARG_STORE yypParser->pCxt =pCxt ; -#define NewParseCTX_SDECL -#define NewParseCTX_PDECL -#define NewParseCTX_PARAM -#define NewParseCTX_FETCH -#define NewParseCTX_STORE +#define ParseARG_SDECL SAstCreateContext* pCxt ; +#define ParseARG_PDECL , SAstCreateContext* pCxt +#define ParseARG_PARAM ,pCxt +#define ParseARG_FETCH SAstCreateContext* pCxt =yypParser->pCxt ; +#define ParseARG_STORE yypParser->pCxt =pCxt ; +#define ParseCTX_SDECL +#define ParseCTX_PDECL +#define ParseCTX_PARAM +#define ParseCTX_FETCH +#define ParseCTX_STORE #define YYNSTATE 278 #define YYNRULE 236 #define YYNTOKEN 134 @@ -544,8 +544,8 @@ struct yyParser { #ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ #endif - NewParseARG_SDECL /* A place to hold %extra_argument */ - NewParseCTX_SDECL /* A place to hold %extra_context */ + ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ @@ -581,7 +581,7 @@ static char *yyTracePrompt = 0; ** Outputs: ** None. */ -void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ +void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ yyTraceFILE = TraceFILE; yyTracePrompt = zTracePrompt; if( yyTraceFILE==0 ) yyTracePrompt = 0; @@ -1082,7 +1082,7 @@ static int yyGrowStack(yyParser *p){ #endif /* Datatype of the argument to the memory allocated passed as the -** second argument to NewParseAlloc() below. This can be changed by +** second argument to ParseAlloc() below. This can be changed by ** putting an appropriate #define in the %include section of the input ** grammar. */ @@ -1092,9 +1092,9 @@ static int yyGrowStack(yyParser *p){ /* Initialize a new parser that has already been allocated. */ -void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ +void ParseInit(void *yypRawParser ParseCTX_PDECL){ yyParser *yypParser = (yyParser*)yypRawParser; - NewParseCTX_STORE + ParseCTX_STORE #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif @@ -1118,7 +1118,7 @@ void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ #endif } -#ifndef NewParse_ENGINEALWAYSONSTACK +#ifndef Parse_ENGINEALWAYSONSTACK /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -1129,18 +1129,18 @@ void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ ** ** Outputs: ** A pointer to a parser. This pointer is used in subsequent calls -** to NewParse and NewParseFree. +** to Parse and ParseFree. */ -void *NewParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) NewParseCTX_PDECL){ +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ yyParser *yypParser; yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( yypParser ){ - NewParseCTX_STORE - NewParseInit(yypParser NewParseCTX_PARAM); + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); } return (void*)yypParser; } -#endif /* NewParse_ENGINEALWAYSONSTACK */ +#endif /* Parse_ENGINEALWAYSONSTACK */ /* The following function deletes the "minor type" or semantic value @@ -1155,8 +1155,8 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1321,7 +1321,7 @@ static void yy_pop_parser_stack(yyParser *pParser){ /* ** Clear all secondary memory allocations from the parser */ -void NewParseFinalize(void *p){ +void ParseFinalize(void *p){ yyParser *pParser = (yyParser*)p; while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 @@ -1329,7 +1329,7 @@ void NewParseFinalize(void *p){ #endif } -#ifndef NewParse_ENGINEALWAYSONSTACK +#ifndef Parse_ENGINEALWAYSONSTACK /* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. @@ -1338,23 +1338,23 @@ void NewParseFinalize(void *p){ ** is defined in a %include section of the input grammar) then it is ** assumed that the input pointer is never NULL. */ -void NewParseFree( +void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ #ifndef YYPARSEFREENEVERNULL if( p==0 ) return; #endif - NewParseFinalize(p); + ParseFinalize(p); (*freeProc)(p); } -#endif /* NewParse_ENGINEALWAYSONSTACK */ +#endif /* Parse_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. */ #ifdef YYTRACKMAXSTACKDEPTH -int NewParseStackPeak(void *p){ +int ParseStackPeak(void *p){ yyParser *pParser = (yyParser*)p; return pParser->yyhwm; } @@ -1378,7 +1378,7 @@ static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; ** Return the number of missed state/lookahead combinations. */ #if defined(YYCOVERAGE) -int NewParseCoverage(FILE *out){ +int ParseCoverage(FILE *out){ int stateno, iLookAhead, i; int nMissed = 0; for(stateno=0; statenoyytos++; @@ -1840,14 +1840,14 @@ static YYACTIONTYPE yy_reduce( yyParser *yypParser, /* The parser */ unsigned int yyruleno, /* Number of the rule by which to reduce */ int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - NewParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - NewParseCTX_PDECL /* %extra_context */ + ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - NewParseARG_FETCH + ParseARG_FETCH (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; @@ -2680,8 +2680,8 @@ static YYACTIONTYPE yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); @@ -2692,8 +2692,8 @@ static void yy_parse_failed( ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - NewParseCTX_STORE + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } #endif /* YYNOERRORRECOVERY */ @@ -2703,10 +2703,10 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - NewParseTOKENTYPE yyminor /* The minor type of the error token */ + ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ @@ -2717,8 +2717,8 @@ static void yy_syntax_error( } pCxt->valid = false; /************ End %syntax_error code ******************************************/ - NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - NewParseCTX_STORE + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* @@ -2727,8 +2727,8 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); @@ -2742,13 +2742,13 @@ static void yy_accept( ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - NewParseCTX_STORE + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* The main parser program. ** The first argument is a pointer to a structure obtained from -** "NewParseAlloc" which describes the current state of the parser. +** "ParseAlloc" which describes the current state of the parser. ** The second argument is the major token number. The third is ** the minor token. The fourth optional argument is whatever the ** user wants (and specified in the grammar) and is available for @@ -2765,11 +2765,11 @@ static void yy_accept( ** Outputs: ** None. */ -void NewParse( +void Parse( void *yyp, /* The parser */ int yymajor, /* The major token code number */ - NewParseTOKENTYPE yyminor /* The value for the token */ - NewParseARG_PDECL /* Optional %extra_argument parameter */ + ParseTOKENTYPE yyminor /* The value for the token */ + ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; YYACTIONTYPE yyact; /* The parser action. */ @@ -2780,8 +2780,8 @@ void NewParse( int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif yyParser *yypParser = (yyParser*)yyp; /* The parser */ - NewParseCTX_FETCH - NewParseARG_STORE + ParseCTX_FETCH + ParseARG_STORE assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) @@ -2806,7 +2806,7 @@ void NewParse( yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor NewParseCTX_PARAM); + yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY @@ -2939,7 +2939,7 @@ void NewParse( ** Return the fallback token corresponding to canonical token iToken, or ** 0 if iToken has no fallback. */ -int NewParseFallback(int iToken){ +int ParseFallback(int iToken){ #ifdef YYFALLBACK if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ return yyFallback[iToken]; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index b025426c87..37784105a4 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -43,11 +43,11 @@ public: SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; - vgroup.epset.eps[0] = (SEp){"dnode_1", 6030}; - vgroup.epset.eps[1] = (SEp){"dnode_2", 6030}; - vgroup.epset.eps[2] = (SEp){"dnode_3", 6030}; - vgroup.epset.inUse = 0; - vgroup.epset.numOfEps = 3; + vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030}; + vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030}; + vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030}; + vgroup.epSet.inUse = 0; + vgroup.epSet.numOfEps = 3; meta_->vgs.emplace_back(vgroup); return *this; @@ -123,14 +123,14 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo vgInfo->vgId = 1; - addEpIntoEpSet(&vgInfo->epset, "node1", 6030); + addEpIntoEpSet(&vgInfo->epSet, "node1", 6030); return 0; } int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const { SVgroupInfo info = {0}; info.vgId = 1; - addEpIntoEpSet(&info.epset, "node1", 6030); + addEpIntoEpSet(&info.epSet, "node1", 6030); info.hashBegin = 0; info.hashEnd = 1; @@ -157,10 +157,10 @@ public: meta_[db][tbname]->schema->uid = id_++; SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; - addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030); - addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030); - addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030); - vgroup.epset.inUse = 0; + addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); + vgroup.epSet.inUse = 0; meta_[db][tbname]->vgs.emplace_back(vgroup); // super table @@ -331,4 +331,4 @@ int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, S int32_t MockCatalogService::catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const { return impl_->catalogGetTableDistVgInfo(pTableName, pVgList); -} \ No newline at end of file +} diff --git a/source/libs/parser/test/parserTest.cpp b/source/libs/parser/test/parserAstTest.cpp similarity index 99% rename from source/libs/parser/test/parserTest.cpp rename to source/libs/parser/test/parserAstTest.cpp index d154ed09a8..4d29475ad7 100644 --- a/source/libs/parser/test/parserTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -18,7 +18,7 @@ #include -#include "parserInt.h" +#include "parInt.h" using namespace std; using namespace testing; diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/parserInsertTest.cpp similarity index 99% rename from source/libs/parser/test/insertParserTest.cpp rename to source/libs/parser/test/parserInsertTest.cpp index 7c46981028..3d4a6e0eb8 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -15,8 +15,7 @@ #include -#include "insertParser.h" -// #include "mockCatalog.h" +#include "parInt.h" using namespace std; using namespace testing; diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp deleted file mode 100644 index c7225639c9..0000000000 --- a/source/libs/parser/test/tokenizerTest.cpp +++ /dev/null @@ -1,730 +0,0 @@ -#if 0 - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include "os.h" - -#include "taos.h" -#include "tvariant.h" -#include "tdef.h" -#include "ttoken.h" -#include "astGenerator.h" -#include "parserUtil.h" -#include "parserInt.h" - -namespace { -int32_t testValidateName(char* name) { - SToken token = {0}; - token.z = name; - token.n = strlen(name); - token.type = 0; - - tGetToken(name, &token.type); - return parserValidateIdToken(&token); -} - -SToken createToken(char* s) { - SToken t = {0}; - - t.type = TK_STRING; - t.z = s; - t.n = strlen(s); - return t; -} -} // namespace - -static void _init_tvariant_bool(SVariant* t) { - t->i = TSDB_FALSE; - t->nType = TSDB_DATA_TYPE_BOOL; -} - -static void _init_tvariant_tinyint(SVariant* t) { - t->i = -27; - t->nType = TSDB_DATA_TYPE_TINYINT; -} - -static void _init_tvariant_int(SVariant* t) { - t->i = -23997659; - t->nType = TSDB_DATA_TYPE_INT; -} - -static void _init_tvariant_bigint(SVariant* t) { - t->i = -3333333333333; - t->nType = TSDB_DATA_TYPE_BIGINT; -} - -static void _init_tvariant_float(SVariant* t) { - t->d = -8991212199.8987878776; - t->nType = TSDB_DATA_TYPE_FLOAT; -} - -static void _init_tvariant_binary(SVariant* t) { - taosVariantDestroy(t); - - t->pz = (char*)calloc(1, 20); //"2e3"); - t->nType = TSDB_DATA_TYPE_BINARY; - strcpy(t->pz, "2e5"); - t->nLen = strlen(t->pz); -} - -static void _init_tvariant_nchar(SVariant* t) { - taosVariantDestroy(t); - - t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); - t->nType = TSDB_DATA_TYPE_NCHAR; - wcscpy(t->wpz, L"-2000000.8765"); - t->nLen = twcslen(t->wpz); -} - -TEST(testCase, validateToken_test) { - char t01[] = "abc"; - EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); - - char t110[] = "`1233abc.911`"; - EXPECT_EQ(testValidateName(t110), TSDB_CODE_SUCCESS); - - char t02[] = "'abc'"; - EXPECT_EQ(testValidateName(t02), TSDB_CODE_TSC_INVALID_OPERATION); - - char t1[] = "abc.def"; - EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); - printf("%s\n", t1); - - char t98[] = "abc.DeF"; - EXPECT_EQ(testValidateName(t98), TSDB_CODE_SUCCESS); - EXPECT_STREQ(t98, "abc.def"); - printf("%s\n", t98); - - char t97[] = "257.abc"; - EXPECT_EQ(testValidateName(t97), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t97); - - char t96[] = "_257.aBc"; - EXPECT_EQ(testValidateName(t96), TSDB_CODE_SUCCESS); - EXPECT_STREQ(t96, "_257.abc"); - printf("%s\n", t96); - - char t99[] = "abc . def"; - EXPECT_EQ(testValidateName(t99), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t99); - - char t2[] = "'abc.def'"; - EXPECT_EQ(testValidateName(t2), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t2); - - char t3[] = "'abc'.def"; - EXPECT_EQ(testValidateName(t3), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t3); - - char t4[] = "'abc'.'def'"; - EXPECT_EQ(testValidateName(t4), TSDB_CODE_TSC_INVALID_OPERATION); - - char t5[] = "table.'def'"; - EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION); - - char t6[] = "'table'.'def'"; - EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION); - - char t7[] = "'_ab1234'.'def'"; - EXPECT_EQ(testValidateName(t7), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t7); - - char t8[] = "'_ab&^%1234'.'def'"; - EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION); - - char t9[] = "'_123'.'gtest中文'"; - EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10[] = "abc.'gtest中文'"; - EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10_1[] = "abc.'中文gtest'"; - EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t11[] = "'192.168.0.1'.abc"; - EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION); - - char t12[] = "192.168.0.1.abc"; - EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION); - - char t13[] = "abc."; - EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION); - - char t14[] = ".abc"; - EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION); - - char t15[] = ".'abc'"; - EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION); - - char t16[] = ".abc'"; - EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION); - - char t17[] = "123a.\"abc\""; - EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t17); - - char t18[] = "a.\"abc\""; - EXPECT_EQ(testValidateName(t18), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t18); - - char t19[] = "'_ab1234'.'def'.'ab123'"; - EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION); - - char t20[] = "'_ab1234*&^'"; - EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION); - - char t21[] = "'1234_abc'"; - EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION); - - // =======Containing capital letters================= - char t30[] = "ABC"; - EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); - - char t31[] = "'ABC'"; - EXPECT_EQ(testValidateName(t31), TSDB_CODE_TSC_INVALID_OPERATION); - - char t32[] = "ABC.def"; - EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); - - char t33[] = "'ABC.def"; - EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION); - - char t33_0[] = "abc.DEF'"; - EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t34[] = "'ABC.def'"; - // int32_t tmp0 = testValidateName(t34); - EXPECT_EQ(testValidateName(t34), TSDB_CODE_TSC_INVALID_OPERATION); - - char t35[] = "'ABC'.def"; - EXPECT_EQ(testValidateName(t35), TSDB_CODE_TSC_INVALID_OPERATION); - - char t36[] = "ABC.DEF"; - EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); - - char t37[] = "abc.DEF"; - EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); - - char t37_1[] = "abc._123DEF"; - EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); - - char t38[] = "'abc'.\"DEF\""; - EXPECT_EQ(testValidateName(t38), TSDB_CODE_TSC_INVALID_OPERATION); - - // do not use key words - char t39[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION); - - char t40[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION); - - char t41[] = "'_abXYZ1234'.'deFF'"; - EXPECT_EQ(testValidateName(t41), TSDB_CODE_TSC_INVALID_OPERATION); - - char t42[] = "'_abDEF&^%1234'.'DIef'"; - EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION); - - char t43[] = "'_123'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION); - - char t44[] = "'aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION); - - char t45[] = "'ABC'."; - EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION); - - char t46[] = ".'ABC'"; - EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION); - - char t47[] = "a.\"aTWc\""; - EXPECT_EQ(testValidateName(t47), TSDB_CODE_TSC_INVALID_OPERATION); - - // ================has space ================= - char t60[] = " ABC "; - EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION); - - char t60_1[] = " ABC "; - EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t62[] = " ABC . def "; - EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63[] = "' ABC . def "; - EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63_0[] = " abc . DEF ' "; - EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t64[] = " ' ABC . def ' "; - // int32_t tmp1 = testValidateName(t64); - EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION); - - char t65[] = " ' ABC '. def "; - EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION); - - char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION); - - char t67[] = "abc . ' DEF '"; - EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION); - - char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION); - - // do not use key words - char t69[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION); - - char t70[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION); - - char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION); - - char t72[] = "'_abDEF&^%1234'.' DIef'"; - EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION); - - char t73[] = "'_123'.' Gtest中文'"; - EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION); - - char t74[] = "' aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION); - - char t75[] = "' ABC '."; - EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION); - - char t76[] = ".' ABC'"; - EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION); - - char t77[] = " a . \"aTWc\" "; - EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION); - - char t78[] = " a.\"aTWc \""; - EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION); - - // ===============muti string by space =================== - // There's no such case. - // char t160[] = "A BC"; - // EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION); - // printf("end:%s\n", t160); - - // There's no such case. - // char t161[] = "' A BC '"; - // EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION); - - char t162[] = " AB C . de f "; - EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163[] = "' AB C . de f "; - EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163_0[] = " ab c . DE F ' "; - EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t164[] = " ' AB C . de f ' "; - // int32_t tmp2 = testValidateName(t164); - EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION); - - char t165[] = " ' A BC '. de f "; - EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION); - - char t166[] = "' AB C '.' DE F '"; - EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION); - - char t167[] = "ab c . ' D EF '"; - EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION); - - char t168[] = "' a bc '.' DE F '"; - EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION); -} - -#if 0 -TEST(testCase, tvariant_convert) { - // 1. bool data to all other data types - SVariant t = {0}; - _init_tvariant_bool(&t); - - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "FALSE"); - taosVariantDestroy(&t); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"FALSE"); - taosVariantDestroy(&t); - - // 2. tinyint to other data types - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-27"); - taosVariantDestroy(&t); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-27"); - taosVariantDestroy(&t); - - // 3. int to other data - // types////////////////////////////////////////////////////////////////// - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-23997659"); - taosVariantDestroy(&t); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-23997659"); - taosVariantDestroy(&t); - - // 4. bigint to other data - // type////////////////////////////////////////////////////////////////////////////// - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-3333333333333"); - taosVariantDestroy(&t); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-3333333333333"); - taosVariantDestroy(&t); - - // 5. float to other data - // types//////////////////////////////////////////////////////////////////////// - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -8991212199); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.d, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.d, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-8991212199.898788"); - taosVariantDestroy(&t); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-8991212199.898788"); - taosVariantDestroy(&t); - - // 6. binary to other data types - // ////////////////////////////////////////////////////////////////// - t.pz = "true"; - t.nLen = strlen(t.pz); - t.nType = TSDB_DATA_TYPE_BINARY; - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.d, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.d, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "2e5"); - taosVariantDestroy(&t); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"2e5"); - taosVariantDestroy(&t); - - // 7. nchar to other data types - // ////////////////////////////////////////////////////////////////// - t.wpz = L"FALSE"; - t.nLen = wcslen(t.wpz); - t.nType = TSDB_DATA_TYPE_NCHAR; - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_nchar(&t); - EXPECT_LE(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -2000000); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.d, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.d, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-2000000.8765"); - taosVariantDestroy(&t); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-2000000.8765"); - taosVariantDestroy(&t); -} -#endif - -TEST(testCase, tGetToken_Test) { - char* s = ".123 "; - uint32_t type = 0; - - int32_t len = tGetToken(s, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s) - 1); - - char s1[] = "1.123e10 "; - len = tGetToken(s1, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s1) - 1); - - char s4[] = "0xff "; - len = tGetToken(s4, &type); - EXPECT_EQ(type, TK_HEX); - EXPECT_EQ(len, strlen(s4) - 1); - - // invalid data type - char s2[] = "e10 "; - len = tGetToken(s2, &type); - EXPECT_FALSE(type == TK_FLOAT); - - char s3[] = "1.1.1.1"; - len = tGetToken(s3, &type); - EXPECT_EQ(type, TK_IPTOKEN); - EXPECT_EQ(len, strlen(s3)); - - char s5[] = "0x "; - len = tGetToken(s5, &type); - EXPECT_FALSE(type == TK_HEX); -} - -TEST(testCase, isValidNumber_test) { - SToken t1 = createToken("123abc"); - - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("0xabc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createToken("0b11101"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createToken(".134abc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("1e1 "); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("1+2"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("-0x123"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createToken("-1"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER); - - t1 = createToken("-0b1110"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createToken("-.234"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT); -} - -TEST(testCase, generateAST_test) { - SSqlInfo info = doGenerateAST("select * from t1 where ts < now"); - ASSERT_EQ(info.valid, true); - - SSqlInfo info1 = doGenerateAST("select * from `t.1abc` where ts. */ -#include "plannerInt.h" +#include "planInt.h" #include "functionMgt.h" diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/planPhysiCreater.c similarity index 75% rename from source/libs/planner/src/physicalPlan.c rename to source/libs/planner/src/planPhysiCreater.c index 61c5f517a7..5327cf89c5 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "plannerInt.h" +#include "planInt.h" #include "functionMgt.h" @@ -28,6 +28,7 @@ typedef struct SPhysiPlanContext { int16_t nextDataBlockId; SArray* pLocationHelper; SArray* pExecNodeList; + int32_t subplanId; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, char* pKey) { @@ -47,7 +48,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_ pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = false; - pSlot->output = false; + pSlot->output = true; return (SNode*)pSlot; } @@ -81,12 +82,7 @@ static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SData SNode* pNode = NULL; int16_t slotId = taosHashGetSize(pHash); FOREACH(pNode, pList) { - SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); - CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(pDataBlockDesc->pSlots, (SNode*)pSlot)) { - nodesDestroyNode(pSlot); - return TSDB_CODE_OUT_OF_MEMORY; - } + CHECK_CODE_EXT(nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId))); SSlotIndex index = { .dataBlockId = pDataBlockDesc->dataBlockId, .slotId = slotId }; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; @@ -97,7 +93,7 @@ static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SData CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); REPLACE_NODE(pTarget); - pDataBlockDesc->resultRowSize += ((SSlotDescNode*)pSlot)->dataType.bytes; + pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; ++slotId; } return TSDB_CODE_SUCCESS; @@ -119,6 +115,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { pIndex = taosHashGet(pCxt->pRightHash, name, len); } // pIndex is definitely not NULL, otherwise it is a bug + CHECK_ALLOC(pIndex, DEAL_RES_ERROR); ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; ((SColumnNode*)pNode)->slotId = pIndex->slotId; CHECK_ALLOC(pNode, DEAL_RES_ERROR); @@ -181,6 +178,8 @@ static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, SData FOREACH(pNode, pTargets) { int32_t len = getSlotKey(pNode, name); SSlotIndex* pIndex = taosHashGet(pHash, name, len); + // pIndex is definitely not NULL, otherwise it is a bug + CHECK_ALLOC(pIndex, TSDB_CODE_FAILED); ((SSlotDescNode*)nodesListGetNode(pDataBlockDesc->pSlots, pIndex->slotId))->output = true; } @@ -195,32 +194,30 @@ static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { pCol->tableId = tableId; pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; pCol->colType = COLUMN_TYPE_COLUMN; + strcpy(pCol->colName, "#primarykey"); return pCol; } -static int32_t addPrimaryKeyCol(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode) { - if (NULL == pScanPhysiNode->pScanCols) { - pScanPhysiNode->pScanCols = nodesMakeList(); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); - return TSDB_CODE_SUCCESS; - } - SNode* pNode; - FOREACH(pNode, pScanPhysiNode->pScanCols) { - if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { - return TSDB_CODE_SUCCESS; - } - } +static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { + pScanPhysiNode->pScanCols = nodesMakeList(); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); + + SNode* pNode; + FOREACH(pNode, pScanCols) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { + SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0); + strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias); + strcpy(pCol->colName, ((SColumnNode*)pNode)->colName); + continue; + } + CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))); + } return TSDB_CODE_SUCCESS; } static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - if (NULL != pScanLogicNode->pScanCols) { - pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - } - CHECK_CODE(addPrimaryKeyCol(pCxt, pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE(createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); @@ -240,7 +237,7 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) { pNodeAddr->nodeId = vg->vgId; - pNodeAddr->epset = vg->epset; + pNodeAddr->epSet = vg->epSet; } static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { @@ -467,6 +464,14 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return (SPhysiNode*)pProject; } +static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + CHECK_ALLOC(pExchange, NULL); + CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); + pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; + return (SPhysiNode*)pExchange; +} + static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SLogicNode* pLogicPlan) { SNodeList* pChildren = nodesMakeList(); CHECK_ALLOC(pChildren, NULL); @@ -495,9 +500,16 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, case QUERY_NODE_LOGIC_PLAN_PROJECT: pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); break; + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + pPhyNode = createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicPlan); + break; default: break; } + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + nodesDestroyNode(pPhyNode); + return NULL; + } pPhyNode->pChildren = pChildren; SNode* pChild; @@ -525,24 +537,44 @@ static SDataSinkNode* createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysi return (SDataSinkNode*)pDispatcher; } +static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { + SSubplan* pSubplan = nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + CHECK_ALLOC(pSubplan, NULL); + pSubplan->id = pLogicSubplan->id; + pSubplan->subplanType = pLogicSubplan->subplanType; + pSubplan->level = pLogicSubplan->level; + return pSubplan; +} + static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { - SSubplan* pSubplan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan); CHECK_ALLOC(pSubplan, NULL); if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) { SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; pSubplan->pDataSink = createDataInserter(pCxt, pModif->pVgDataBlocks); pSubplan->msgType = pModif->msgType; - pSubplan->execNode.epset = pModif->pVgDataBlocks->vg.epset; + pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet; taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode); pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); pSubplan->msgType = TDMT_VND_QUERY; } - pSubplan->subplanType = pLogicSubplan->subplanType; return pSubplan; } +static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { + pNode->pParent = pParent; + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + doSetLogicNodeParent((SLogicNode*)pChild, pNode); + } +} + +static void setLogicNodeParent(SLogicNode* pNode) { + doSetLogicNodeParent(pNode, NULL); +} + static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubLogicPlan** pSubLogicPlan) { *pSubLogicPlan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); CHECK_ALLOC(*pSubLogicPlan, TSDB_CODE_OUT_OF_MEMORY); @@ -553,8 +585,9 @@ static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, S } else { (*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_MERGE; } - // todo split - return TSDB_CODE_SUCCESS; + (*pSubLogicPlan)->id.queryId = pCxt->pPlanCxt->queryId; + setLogicNodeParent((*pSubLogicPlan)->pNode); + return applySplitRule(*pSubLogicPlan); } static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t level, SNodeList* pSubplans) { @@ -571,9 +604,10 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l CHECK_ALLOC(pGroup->pNodeList, TSDB_CODE_OUT_OF_MEMORY); } CHECK_CODE(nodesListStrictAppend(pGroup->pNodeList, pSubplan), TSDB_CODE_OUT_OF_MEMORY); + return TSDB_CODE_SUCCESS; } -SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { +static SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { SSubLogicPlan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); CHECK_ALLOC(pDst, NULL); pDst->pNode = nodesCloneNode(pSrc->pNode); @@ -583,29 +617,120 @@ SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* p } pDst->subplanType = pSrc->subplanType; pDst->level = level; + pDst->id.queryId = pSrc->id.queryId; + pDst->id.groupId = pSrc->id.groupId; + pDst->id.subplanId = pCxt->subplanId++; return pDst; } -static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan) { - if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) { - SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; - size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); - for (int32_t i = 0; i < numOfVgroups; ++i) { - SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); - ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; - CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); - } - } else { +static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { + SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; + size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); + for (int32_t i = 0; i < numOfVgroups; ++i) { SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); + SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); + ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; + CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t scaleOutForMerge(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { + return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); +} + +static int32_t doSetScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + CHECK_ALLOC(pScan->pVgroupList, TSDB_CODE_OUT_OF_MEMORY); + memcpy(pScan->pVgroupList->vgroups, pVgroup, sizeof(SVgroupInfo)); + *pFound = true; + return TSDB_CODE_SUCCESS; + } + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + int32_t code = doSetScanVgroup(pCxt, (SLogicNode*)pChild, pVgroup, pFound); + if (TSDB_CODE_SUCCESS != code || *pFound) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup) { + bool found = false; + return doSetScanVgroup(pCxt, pNode, pVgroup, &found); +} + +static int32_t scaleOutForScan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { + if (pSubplan->pVgroupList) { + for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { + SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE_EXT(setScanVgroup(pCxt, pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i)); + CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); + } + return TSDB_CODE_SUCCESS; + } else { + return scaleOutForMerge(pCxt, pSubplan, level, pGroup); + } +} + +static int32_t appendWithMakeList(SNodeList** pList, SNodeptr pNode) { + if (NULL == *pList) { + *pList = nodesMakeList(); + if (NULL == *pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return nodesListAppend(*pList, pNode); +} + +static int32_t pushHierarchicalPlan(SPhysiPlanContext* pCxt, SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { + bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); + SNode* pChild = NULL; + FOREACH(pChild, pCurrentGroup) { + if (topLevel) { + CHECK_CODE_EXT(nodesListAppend(pParentsGroup, pChild)); + } else { + SNode* pParent = NULL; + FOREACH(pParent, pParentsGroup) { + CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pParent)->pChildren), pChild)); + CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pChild)->pParents), pParent)); + } + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { + SNodeList* pCurrentGroup = nodesMakeList(); + CHECK_ALLOC(pCurrentGroup, TSDB_CODE_OUT_OF_MEMORY); + int32_t code = TSDB_CODE_SUCCESS; + switch (pSubplan->subplanType) { + case SUBPLAN_TYPE_MERGE: + code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + case SUBPLAN_TYPE_SCAN: + code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + case SUBPLAN_TYPE_MODIFY: + code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + default: + break; + } + if (TSDB_CODE_SUCCESS != code) { + return code; } + CHECK_CODE_EXT(pushHierarchicalPlan(pCxt, pParentsGroup, pCurrentGroup)); + ++(*pLevel); SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, level + 1, pLogicPlan)); + CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, pLevel, pCurrentGroup)); } return TSDB_CODE_SUCCESS; @@ -614,8 +739,8 @@ static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int3 static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) { SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); CHECK_ALLOC(pLogicPlan, NULL); - pLogicPlan->pSubplans = nodesMakeList(); - if (NULL == pLogicPlan->pSubplans) { + pLogicPlan->pTopSubplans = nodesMakeList(); + if (NULL == pLogicPlan->pTopSubplans) { nodesDestroyNode(pLogicPlan); return NULL; } @@ -625,25 +750,7 @@ static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) { static int32_t scaleOutLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pRootSubLogicPlan, SQueryLogicPlan** pLogicPlan) { *pLogicPlan = makeQueryLogicPlan(pCxt); CHECK_ALLOC(*pLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - return doScaleOut(pCxt, pRootSubLogicPlan, 0, *pLogicPlan); -} - -typedef struct SBuildPhysiSubplanCxt { - int32_t errCode; - SQueryPlan* pQueryPlan; - SPhysiPlanContext* pPhyCxt; -} SBuildPhysiSubplanCxt; - -static EDealRes doBuildPhysiSubplan(SNode* pNode, void* pContext) { - SBuildPhysiSubplanCxt* pCxt = (SBuildPhysiSubplanCxt*)pContext; - if (QUERY_NODE_LOGIC_SUBPLAN == nodeType(pNode)) { - SSubplan* pSubplan = createPhysiSubplan(pCxt->pPhyCxt, (SSubLogicPlan*)pNode); - CHECK_ALLOC(pSubplan, DEAL_RES_ERROR); - CHECK_CODE(pushSubplan(pCxt->pPhyCxt, pSubplan, ((SSubLogicPlan*)pNode)->level, pCxt->pQueryPlan->pSubplans), DEAL_RES_ERROR); - ++(pCxt->pQueryPlan->numOfSubplans); - return DEAL_RES_IGNORE_CHILD; - } - return DEAL_RES_CONTINUE; + return doScaleOut(pCxt, pRootSubLogicPlan, &((*pLogicPlan)->totalLevel), (*pLogicPlan)->pTopSubplans); } static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { @@ -658,15 +765,31 @@ static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { return pPlan; } -static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { - SBuildPhysiSubplanCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pQueryPlan = makeQueryPhysiPlan(pCxt), .pPhyCxt = pCxt }; - CHECK_ALLOC(cxt.pQueryPlan, TSDB_CODE_OUT_OF_MEMORY); - nodesWalkList(pLogicPlan->pSubplans, doBuildPhysiSubplan, &cxt); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyNode(cxt.pQueryPlan); - return cxt.errCode; +static int32_t doBuildPhysiPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { + SSubplan* pSubplan = createPhysiSubplan(pCxt, pLogicSubplan); + CHECK_ALLOC(pSubplan, DEAL_RES_ERROR); + CHECK_CODE_EXT(pushSubplan(pCxt, pSubplan, pLogicSubplan->level, pQueryPlan->pSubplans)); + ++(pQueryPlan->numOfSubplans); + if (NULL != pParent) { + CHECK_CODE_EXT(appendWithMakeList(&pParent->pChildren, pSubplan)); + CHECK_CODE_EXT(appendWithMakeList(&pSubplan->pParents, pParent)); + } + + SNode* pChild = NULL; + FOREACH(pChild, pLogicSubplan->pChildren) { + CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pChild, pSubplan, pQueryPlan)); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { + *pPlan = makeQueryPhysiPlan(pCxt); + CHECK_ALLOC(*pPlan, TSDB_CODE_OUT_OF_MEMORY); + SNode* pSubplan = NULL; + FOREACH(pSubplan, pLogicPlan->pTopSubplans) { + CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pSubplan, NULL, *pPlan)); } - *pPlan = cxt.pQueryPlan; return TSDB_CODE_SUCCESS; } @@ -681,13 +804,12 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - SQueryLogicPlan* pLogicPlan; - SSubLogicPlan* pSubLogicPlan; + SQueryLogicPlan* pLogicPlan = NULL; + SSubLogicPlan* pSubLogicPlan = NULL; int32_t code = splitLogicPlan(&cxt, pLogicNode, &pSubLogicPlan); if (TSDB_CODE_SUCCESS == code) { code = scaleOutLogicPlan(&cxt, pSubLogicPlan, &pLogicPlan); } - // todo maping if (TSDB_CODE_SUCCESS == code) { code = buildPhysiPlan(&cxt, pLogicPlan, pPlan); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c new file mode 100644 index 0000000000..5a5e1d46c6 --- /dev/null +++ b/source/libs/planner/src/planSpliter.c @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +#define SPLIT_FLAG_MASK(n) (1 << n) + +#define SPLIT_FLAG_STS SPLIT_FLAG_MASK(0) + +#define SPLIT_FLAG_SET_MASK(val, mask) (val) |= (mask) +#define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) + +typedef struct SSplitContext { + int32_t errCode; + int32_t groupId; + bool match; + void* pInfo; +} SSplitContext; + +typedef int32_t (*FMatch)(SSplitContext* pCxt, SSubLogicPlan* pSubplan); +typedef int32_t (*FSplit)(SSplitContext* pCxt); + +typedef struct SSplitRule { + char* pName; + FMatch matchFunc; + FSplit splitFunc; +} SSplitRule; + +typedef struct SStsInfo { + SScanLogicNode* pScan; + SSubLogicPlan* pSubplan; +} SStsInfo; + +static SLogicNode* stsMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = stsMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { + if (SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { + return TSDB_CODE_SUCCESS; + } + SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + SStsInfo* pInfo = calloc(1, sizeof(SStsInfo)); + CHECK_ALLOC(pInfo, TSDB_CODE_OUT_OF_MEMORY); + pInfo->pScan = (SScanLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + pCxt->pInfo = pInfo; + pCxt->match = true; + return TSDB_CODE_SUCCESS; + } + SNode* pChild; + FOREACH(pChild, pSubplan->pChildren) { + int32_t code = stsMatch(pCxt, (SSubLogicPlan*)pChild); + if (TSDB_CODE_SUCCESS != code || pCxt->match) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { + SSubLogicPlan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return NULL; + } + pSubplan->id.groupId = pCxt->groupId; + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); + TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo); + SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS); + return pSubplan; +} + +static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SSubLogicPlan* pSubplan, SScanLogicNode* pScan) { + SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pCxt->groupId; + pExchange->node.pTargets = nodesCloneList(pScan->node.pTargets); + if (NULL == pExchange->node.pTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL == pScan->node.pParent) { + pSubplan->pNode = (SLogicNode*)pExchange; + return TSDB_CODE_SUCCESS; + } + + SNode* pNode; + FOREACH(pNode, pScan->node.pParent->pChildren) { + if (nodesEqualNode(pNode, pScan)) { + REPLACE_NODE(pExchange); + nodesDestroyNode(pNode); + return TSDB_CODE_SUCCESS; + } + } + nodesDestroyNode(pExchange); + return TSDB_CODE_FAILED; +} + +static int32_t stsSplit(SSplitContext* pCxt) { + SStsInfo* pInfo = pCxt->pInfo; + if (NULL == pInfo->pSubplan->pChildren) { + pInfo->pSubplan->pChildren = nodesMakeList(); + if (NULL == pInfo->pSubplan->pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + int32_t code = nodesListStrictAppend(pInfo->pSubplan->pChildren, stsCreateScanSubplan(pCxt, pInfo->pScan)); + if (TSDB_CODE_SUCCESS == code) { + code = stsCreateExchangeNode(pCxt, pInfo->pSubplan, pInfo->pScan); + } + ++(pCxt->groupId); + return code; +} + +static const SSplitRule splitRuleSet[] = { + { .pName = "SuperTableScan", .matchFunc = stsMatch, .splitFunc = stsSplit } +}; + +static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); + +int32_t applySplitRule(SSubLogicPlan* pSubplan) { + SSplitContext cxt = { .errCode = TSDB_CODE_SUCCESS, .groupId = pSubplan->id.groupId + 1, .match = false, .pInfo = NULL }; + bool split = false; + do { + split = false; + for (int32_t i = 0; i < splitRuleNum; ++i) { + cxt.match = false; + int32_t code = splitRuleSet[i].matchFunc(&cxt, pSubplan); + if (TSDB_CODE_SUCCESS == code && cxt.match) { + code = splitRuleSet[i].splitFunc(&cxt); + split = true; + } + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } while (split); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index d7d9f1d129..fa0dc549c8 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -15,7 +15,7 @@ #include "planner.h" -#include "plannerInt.h" +#include "planInt.h" int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode) { return TSDB_CODE_SUCCESS; @@ -34,33 +34,68 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo return code; } -void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { +static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDownstreamSourceNode* pSource) { + if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pNode)) { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pNode; + if (pExchange->srcGroupId == groupId) { + if (NULL == pExchange->pSrcEndPoints) { + pExchange->pSrcEndPoints = nodesMakeList(); + if (NULL == pExchange->pSrcEndPoints) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pExchange->pSrcEndPoints, nodesCloneNode(pSource))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; + } + } + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + if (TSDB_CODE_SUCCESS != setSubplanExecutionNode((SPhysiNode*)pChild, groupId, pSource)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return TSDB_CODE_SUCCESS; } -int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len) { - if (SUBPLAN_TYPE_MODIFY == subplan->subplanType) { - SDataInserterNode* insert = (SDataInserterNode*)subplan->pDataSink; - *len = insert->size; - *str = insert->pData; +int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { + return setSubplanExecutionNode(subplan->pNode, groupId, pSource); +} + +int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { + if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) { + SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink; + *pLen = insert->size; + *pStr = insert->pData; insert->pData = NULL; return TSDB_CODE_SUCCESS; } - return nodesNodeToString((const SNode*)subplan, false, str, len); + return nodesNodeToString((const SNode*)pSubplan, false, pStr, pLen); } -int32_t qStringToSubplan(const char* str, SSubplan** subplan) { - return nodesStringToNode(str, (SNode**)subplan); +int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan) { + return nodesStringToNode(pStr, (SNode**)pSubplan); } char* qQueryPlanToString(const SQueryPlan* pPlan) { - + char* pStr = NULL; + int32_t len = 0; + if (TSDB_CODE_SUCCESS != nodesNodeToString(pPlan, false, &pStr, &len)) { + return NULL; + } + return pStr; } SQueryPlan* qStringToQueryPlan(const char* pStr) { - + SQueryPlan* pPlan = NULL; + if (TSDB_CODE_SUCCESS != nodesStringToNode(pStr, (SNode**)&pPlan)) { + return NULL; + } + return pPlan; } void qDestroyQueryPlan(SQueryPlan* pPlan) { - + nodesDestroyNode(pPlan); } diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 8ba80d1ab9..3748d37d74 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -18,7 +18,7 @@ #include #include "parser.h" -#include "plannerInt.h" +#include "planInt.h" using namespace std; using namespace testing; @@ -137,6 +137,13 @@ TEST_F(PlannerTest, simple) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, stSimple) { + setDatabase("root", "test"); + + bind("SELECT * FROM st1"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, groupBy) { setDatabase("root", "test"); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index a1e3f82d4c..5f83720f77 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -266,7 +266,7 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTas int32_t idx = abs((++qwtTestCaseIdx) % qwtTestCaseNum); qwtTestSinkBlockNum = 0; - qwtTestSinkMaxBlockNum = rand() % 100 + 1; + qwtTestSinkMaxBlockNum = taosRand() % 100 + 1; qwtTestSinkQueryEnd = false; if (0 == idx) { @@ -295,15 +295,15 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { } else { if (qwtTestSinkQueryEnd) { *pRes = NULL; - *useconds = rand() % 10; + *useconds = taosRand() % 10; return 0; } - endExec = rand() % 5; + endExec = taosRand() % 5; int32_t runTime = 0; if (qwtTestEnableSleep && qwtTestMaxExecTaskUsec > 0) { - runTime = rand() % qwtTestMaxExecTaskUsec; + runTime = taosRand() % qwtTestMaxExecTaskUsec; } if (qwtTestEnableSleep) { @@ -314,10 +314,10 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (endExec) { *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); - (*pRes)->info.rows = rand() % 1000; + (*pRes)->info.rows = taosRand() % 1000; } else { *pRes = NULL; - *useconds = rand() % 10; + *useconds = taosRand() % 10; } } @@ -376,7 +376,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { taosWLockLatch(&qwtTestSinkLock); if (qwtTestSinkBlockNum > 0) { - *pLen = rand() % 100 + 1; + *pLen = taosRand() % 100 + 1; qwtTestSinkBlockNum--; } else { *pLen = 0; @@ -392,7 +392,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { taosWLockLatch(&qwtTestSinkLock); if (qwtTestSinkLastLen > 0) { - pOutput->numOfRows = rand() % 10 + 1; + pOutput->numOfRows = taosRand() % 10 + 1; pOutput->compressed = 1; pOutput->queryEnd = qwtTestSinkQueryEnd; if (qwtTestSinkBlockNum == 0) { @@ -402,7 +402,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { } else { pOutput->bufStatus = DS_BUF_FULL; } - pOutput->useconds = rand() % 10 + 1; + pOutput->useconds = taosRand() % 10 + 1; pOutput->precision = 1; } else if (qwtTestSinkLastLen == 0) { pOutput->numOfRows = 0; @@ -416,7 +416,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { } else { pOutput->bufStatus = DS_BUF_FULL; } - pOutput->useconds = rand() % 10 + 1; + pOutput->useconds = taosRand() % 10 + 1; pOutput->precision = 1; } else { assert(0); @@ -590,7 +590,7 @@ void *queryThread(void *param) { qwtBuildQueryReqMsg(&queryRpc); qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("query:%d\n", n); @@ -612,7 +612,7 @@ void *readyThread(void *param) { qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("ready:%d\n", n); @@ -634,7 +634,7 @@ void *fetchThread(void *param) { qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("fetch:%d\n", n); @@ -656,7 +656,7 @@ void *dropThread(void *param) { qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("drop:%d\n", n); @@ -678,7 +678,7 @@ void *statusThread(void *param) { qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("status:%d\n", n); @@ -748,7 +748,7 @@ void *queryQueueThread(void *param) { if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { - int32_t delay = rand() % qwtTestReqMaxDelayUsec; + int32_t delay = taosRand() % qwtTestReqMaxDelayUsec; if (delay) { usleep(delay); @@ -804,7 +804,7 @@ void *fetchQueueThread(void *param) { taosWUnLockLatch(&qwtTestFetchQueueLock); if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { - int32_t delay = rand() % qwtTestReqMaxDelayUsec; + int32_t delay = taosRand() % qwtTestReqMaxDelayUsec; if (delay) { usleep(delay); @@ -963,7 +963,7 @@ TEST(seqTest, randCase) { stubSetRpcSendResponse(); stubSetCreateExecTask(); - srand(time(NULL)); + taosSeedRand(time(NULL)); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -971,7 +971,7 @@ TEST(seqTest, randCase) { int32_t t = 0; int32_t maxr = 10001; while (true) { - int32_t r = rand() % maxr; + int32_t r = taosRand() % maxr; if (r >= 0 && r < maxr/5) { printf("Query,%d\n", t++); @@ -1025,7 +1025,7 @@ TEST(seqTest, multithreadRand) { stubSetStringToPlan(); stubSetRpcSendResponse(); - srand(time(NULL)); + taosSeedRand(time(NULL)); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -1076,7 +1076,7 @@ TEST(rcTest, shortExecshortDelay) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); qwtTestStop = false; qwtTestQuitThreadNum = 0; @@ -1157,7 +1157,7 @@ TEST(rcTest, longExecshortDelay) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); qwtTestStop = false; qwtTestQuitThreadNum = 0; @@ -1240,7 +1240,7 @@ TEST(rcTest, shortExeclongDelay) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); qwtTestStop = false; qwtTestQuitThreadNum = 0; @@ -1324,7 +1324,7 @@ TEST(rcTest, dropTest) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -1358,7 +1358,7 @@ TEST(rcTest, dropTest) { int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 18784c8cb5..d435159b2e 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -1286,7 +1286,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 98467e045c..a631511867 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1435,7 +1435,7 @@ TEST(columnTest, greater_and_lower) { int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 146c7c1966..d8ee04ef2b 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -191,8 +191,8 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_NEED_DROP_JOB(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) -#define SCH_GET_CUR_EP(_addr) (&(_addr)->epset.eps[(_addr)->epset.inUse]) -#define SCH_SWITCH_EPSET(_addr) ((_addr)->epset.inUse = ((_addr)->epset.inUse + 1) % (_addr)->epset.numOfEps) +#define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) +#define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) #define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 02eb8869e4..9a69220d7e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -439,13 +439,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pTask->plan->execNode.epset.numOfEps > 0) { + if (pTask->plan->execNode.epSet.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps); + SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -872,8 +872,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); - SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; - qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); + SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, .taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; + qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { @@ -1247,7 +1247,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, isCandidateAddr = true; } - SEpSet epSet = addr->epset; + SEpSet epSet = addr->epSet; switch (msgType) { case TDMT_VND_CREATE_TABLE: diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 0347318ae5..0e2ec9f00d 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -99,13 +99,13 @@ void schtBuildQueryDag(SQueryPlan *dag) { SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); scanPlan->id.queryId = qId; - scanPlan->id.templateId = 0x0000000000000002; + scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003; scanPlan->subplanType = SUBPLAN_TYPE_SCAN; scanPlan->execNode.nodeId = 1; - scanPlan->execNode.epset.inUse = 0; - addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030); + scanPlan->execNode.epSet.inUse = 0; + addEpIntoEpSet(&scanPlan->execNode.epSet, "ep0", 6030); scanPlan->pChildren = NULL; scanPlan->level = 1; @@ -114,11 +114,11 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; - mergePlan->id.templateId = schtMergeTemplateId; + mergePlan->id.groupId = schtMergeTemplateId; mergePlan->id.subplanId = 0x5555; mergePlan->subplanType = SUBPLAN_TYPE_MERGE; mergePlan->level = 0; - mergePlan->execNode.epset.numOfEps = 0; + mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pChildren = nodesMakeList(); mergePlan->pParents = NULL; @@ -158,17 +158,17 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { for (int32_t i = 0; i < scanPlanNum; ++i) { scanPlan[i].id.queryId = qId; - scanPlan[i].id.templateId = 0x0000000000000002; + scanPlan[i].id.groupId = 0x0000000000000002; scanPlan[i].id.subplanId = 0x0000000000000003 + i; scanPlan[i].subplanType = SUBPLAN_TYPE_SCAN; scanPlan[i].execNode.nodeId = 1 + i; - scanPlan[i].execNode.epset.inUse = 0; - scanPlan[i].execNodeStat.tableNum = rand() % 30; - addEpIntoEpSet(&scanPlan[i].execNode.epset, "ep0", 6030); - addEpIntoEpSet(&scanPlan[i].execNode.epset, "ep1", 6030); - addEpIntoEpSet(&scanPlan[i].execNode.epset, "ep2", 6030); - scanPlan[i].execNode.epset.inUse = rand() % 3; + scanPlan[i].execNode.epSet.inUse = 0; + scanPlan[i].execNodeStat.tableNum = taosRand() % 30; + addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep0", 6030); + addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep1", 6030); + addEpIntoEpSet(&scanPlan[i].execNode.epSet, "ep2", 6030); + scanPlan[i].execNode.epSet.inUse = taosRand() % 3; scanPlan[i].pChildren = NULL; scanPlan[i].level = 1; @@ -183,11 +183,11 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { } mergePlan->id.queryId = qId; - mergePlan->id.templateId = schtMergeTemplateId; + mergePlan->id.groupId = schtMergeTemplateId; mergePlan->id.subplanId = 0x5555; mergePlan->subplanType = SUBPLAN_TYPE_MERGE; mergePlan->level = 0; - mergePlan->execNode.epset.numOfEps = 0; + mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); @@ -216,14 +216,14 @@ void schtBuildInsertDag(SQueryPlan *dag) { SSubplan *insertPlan = (SSubplan *)calloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; - insertPlan[0].id.templateId = 0x0000000000000003; + insertPlan[0].id.groupId = 0x0000000000000003; insertPlan[0].id.subplanId = 0x0000000000000004; insertPlan[0].subplanType = SUBPLAN_TYPE_MODIFY; insertPlan[0].level = 0; insertPlan[0].execNode.nodeId = 1; - insertPlan[0].execNode.epset.inUse = 0; - addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030); + insertPlan[0].execNode.epSet.inUse = 0; + addEpIntoEpSet(&insertPlan[0].execNode.epSet, "ep0", 6030); insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; @@ -232,14 +232,14 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; - insertPlan[1].id.templateId = 0x0000000000000003; + insertPlan[1].id.groupId = 0x0000000000000003; insertPlan[1].id.subplanId = 0x0000000000000005; insertPlan[1].subplanType = SUBPLAN_TYPE_MODIFY; insertPlan[1].level = 0; insertPlan[1].execNode.nodeId = 1; - insertPlan[1].execNode.epset.inUse = 0; - addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030); + insertPlan[1].execNode.epSet.inUse = 0; + addEpIntoEpSet(&insertPlan[1].execNode.epSet, "ep0", 6030); insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; @@ -263,7 +263,7 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { return 0; } -void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { +void schtExecNode(SSubplan* subplan, uint64_t groupId, SQueryNodeAddr* ep) { } @@ -599,7 +599,7 @@ void* schtRunJobThread(void *aa) { void* schtFreeJobThread(void *aa) { while (!schtTestStop) { - usleep(rand() % 100); + usleep(taosRand() % 100); schtFreeQueryJob(1); } } @@ -724,7 +724,7 @@ TEST(queryTest, flowCtrlCase) { schtInitLogFile(); - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); @@ -873,7 +873,7 @@ TEST(multiThread, forceFree) { } int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h new file mode 100644 index 0000000000..7116ae9d46 --- /dev/null +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_INDEX_MGR_H +#define _TD_LIBS_SYNC_INDEX_MGR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" + +// SIndexMgr ----------------------------- +typedef struct SSyncIndexMgr { + SRaftId (*replicas)[TSDB_MAX_REPLICA]; + SyncIndex index[TSDB_MAX_REPLICA]; + int32_t replicaNum; + SSyncNode *pSyncNode; +} SSyncIndexMgr; + +SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode); +void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); +void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); +void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); +SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); +cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_INDEX_MGR_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index b8c7eb60e7..8b77e292c4 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "cJSON.h" #include "sync.h" #include "taosdef.h" #include "tglobal.h" @@ -99,8 +100,11 @@ typedef struct SRaftStore SRaftStore; struct SVotesGranted; typedef struct SVotesGranted SVotesGranted; -struct SVotesResponded; -typedef struct SVotesResponded SVotesResponded; +struct SVotesRespond; +typedef struct SVotesRespond SVotesRespond; + +struct SSyncIndexMgr; +typedef struct SSyncIndexMgr SSyncIndexMgr; typedef struct SRaftId { SyncNodeId addr; // typedef uint64_t SyncNodeId; @@ -112,17 +116,18 @@ typedef struct SSyncNode { SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; + char walPath[TSDB_FILENAME_LEN]; void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); void* queue; int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg); // init internal - SNodeInfo me; - SRaftId raftId; + SNodeInfo myNodeInfo; + SRaftId myRaftId; int32_t peersNum; - SNodeInfo peers[TSDB_MAX_REPLICA]; + SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA]; SRaftId peersId[TSDB_MAX_REPLICA]; int32_t replicaNum; @@ -142,18 +147,18 @@ typedef struct SSyncNode { SRaftStore* pRaftStore; // tla+ candidate vars - SVotesGranted* pVotesGranted; - SVotesResponded* pVotesResponded; + SVotesGranted* pVotesGranted; + SVotesRespond* pVotesRespond; // tla+ leader vars - SHashObj* pNextIndex; - SHashObj* pMatchIndex; + SSyncIndexMgr* pNextIndex; + SSyncIndexMgr* pMatchIndex; // tla+ log vars SSyncLogStore* pLogStore; SyncIndex commitIndex; - // timer + // ping timer tmr_h pPingTimer; int32_t pingTimerMS; uint64_t pingTimerLogicClock; @@ -161,6 +166,7 @@ typedef struct SSyncNode { TAOS_TMR_CALLBACK FpPingTimer; // Timer Fp uint64_t pingTimerCounter; + // elect timer tmr_h pElectTimer; int32_t electTimerMS; uint64_t electTimerLogicClock; @@ -168,6 +174,7 @@ typedef struct SSyncNode { TAOS_TMR_CALLBACK FpElectTimer; // Timer Fp uint64_t electTimerCounter; + // heartbeat timer tmr_h pHeartbeatTimer; int32_t heartbeatTimerMS; uint64_t heartbeatTimerLogicClock; @@ -188,6 +195,8 @@ typedef struct SSyncNode { SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 467cfdde5c..aca6205b9d 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -53,6 +53,7 @@ extern "C" { // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); +int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 9b481e82d9..f2c979da99 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -42,8 +42,13 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest); void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest); // ---- misc ---- -int32_t syncUtilRand(int32_t max); -int32_t syncUtilElectRandomMS(); +int32_t syncUtilRand(int32_t max); +int32_t syncUtilElectRandomMS(); +int32_t syncUtilQuorum(int32_t replicaNum); +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); +cJSON* syncUtilRaftId2Json(const SRaftId* p); +char* syncUtilRaftId2Str(const SRaftId* p); +const char* syncUtilState2String(ESyncState state); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index e2307e9e66..ae9cfe8d01 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -28,10 +28,14 @@ extern "C" { #include "syncUtil.h" #include "taosdef.h" +// SVotesGranted ----------------------------- typedef struct SVotesGranted { + SRaftId (*replicas)[TSDB_MAX_REPLICA]; + bool isGranted[TSDB_MAX_REPLICA]; + int32_t replicaNum; + int32_t votes; SyncTerm term; int32_t quorum; - int32_t votes; bool toLeader; SSyncNode *pSyncNode; } SVotesGranted; @@ -41,7 +45,10 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted); bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); +cJSON * voteGranted2Json(SVotesGranted *pVotesGranted); +char * voteGranted2Str(SVotesGranted *pVotesGranted); +// SVotesRespond ----------------------------- typedef struct SVotesRespond { SRaftId (*replicas)[TSDB_MAX_REPLICA]; bool isRespond[TSDB_MAX_REPLICA]; @@ -51,9 +58,12 @@ typedef struct SVotesRespond { } SVotesRespond; SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode); +void votesRespondDestory(SVotesRespond *pVotesRespond); bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); -void Reset(SVotesRespond *pVotesRespond, SyncTerm term); +void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); +cJSON * votesRespond2Json(SVotesRespond *pVotesRespond); +char * votesRespond2Str(SVotesRespond *pVotesRespond); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 87017b718d..223431336e 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -15,6 +15,7 @@ #include "syncElection.h" #include "syncMessage.h" +#include "syncRaftStore.h" // TLA+ Spec // RequestVote(i, j) == @@ -28,11 +29,31 @@ // mdest |-> j]) // /\ UNCHANGED <> // -int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {} +int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SyncRequestVote* pMsg = syncRequestVoteBuild(); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->peersId[i]; + pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm; + pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); + pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); + + ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg); + assert(ret == 0); + syncRequestVoteDestroy(pMsg); + } + return ret; +} int32_t syncNodeElect(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + // start election - syncNodeRequestVotePeers(pSyncNode); + int32_t ret = syncNodeRequestVotePeers(pSyncNode); + return ret; } int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg) { diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 6917df1597..fa58bda76f 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -28,7 +28,7 @@ static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer); int32_t syncEnvStart() { int32_t ret; - srand(time(NULL)); + taosSeedRand(time(NULL)); gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); assert(gSyncEnv != NULL); ret = doSyncEnvStart(gSyncEnv); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index d37c821a24..7cb42c9f87 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -44,7 +44,7 @@ int32_t syncIOStart(char *host, uint16_t port) { gSyncIO = syncIOCreate(host, port); assert(gSyncIO != NULL); - srand(time(NULL)); + taosSeedRand(time(NULL)); int32_t ret = syncIOStartInternal(gSyncIO); assert(ret == 0); diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c new file mode 100644 index 0000000000..fff54638e2 --- /dev/null +++ b/source/libs/sync/src/syncIndexMgr.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncIndexMgr.h" +#include "syncUtil.h" + +// SMatchIndex ----------------------------- + +SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { + SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr)); + assert(pSyncIndexMgr != NULL); + memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); + + pSyncIndexMgr->replicas = &(pSyncNode->replicasId); + pSyncIndexMgr->replicaNum = pSyncNode->replicaNum; + pSyncIndexMgr->pSyncNode = pSyncNode; + syncIndexMgrClear(pSyncIndexMgr); + + return pSyncIndexMgr; +} + +void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { + if (pSyncIndexMgr != NULL) { + free(pSyncIndexMgr); + } +} + +void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { + memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index)); + /* + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + pSyncIndexMgr->index[i] = 0; + } + */ +} + +void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + (pSyncIndexMgr->index)[i] = index; + return; + } + } + assert(0); +} + +SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + SyncIndex idx = (pSyncIndexMgr->index)[i]; + return idx; + } + } + assert(0); +} + +cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncIndexMgr->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); + } + int respondNum = 0; + int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->index[i]; + } + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "index", pIndex); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "pSyncIndexMgr", pRoot); + return pJson; +} + +char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { + cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index db34d16690..3b8d716dbe 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -20,10 +20,12 @@ #include "syncEnv.h" #include "syncInt.h" #include "syncRaft.h" +#include "syncRaftStore.h" #include "syncRequestVote.h" #include "syncRequestVoteReply.h" #include "syncTimeout.h" #include "syncUtil.h" +#include "syncVoteMgr.h" static int32_t tsNodeRefId = -1; @@ -35,6 +37,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); @@ -71,29 +74,60 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); + // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->syncCfg = pSyncInfo->syncCfg; memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); - pSyncNode->pFsm = pSyncInfo->pFsm; - + memcpy(pSyncNode->walPath, pSyncInfo->walPath, sizeof(pSyncNode->walPath)); pSyncNode->rpcClient = pSyncInfo->rpcClient; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; pSyncNode->queue = pSyncInfo->queue; pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg; - pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; - pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; + // init internal + pSyncNode->myNodeInfo = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; + syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncInfo->vgId, &pSyncNode->myRaftId); + // init peersNum, peers, peersId + pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; int j = 0; for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { if (i != pSyncInfo->syncCfg.myIndex) { - pSyncNode->peers[j] = pSyncInfo->syncCfg.nodeInfo[i]; + pSyncNode->peersNodeInfo[j] = pSyncInfo->syncCfg.nodeInfo[i]; j++; } } + for (int i = 0; i < pSyncNode->peersNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncInfo->vgId, &pSyncNode->peersId[i]); + } + // init replicaNum, replicasId + pSyncNode->replicaNum = pSyncInfo->syncCfg.replicaNum; + for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]); + } + + // init raft algorithm + pSyncNode->pFsm = pSyncInfo->pFsm; + pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum); + pSyncNode->leaderCache = EMPTY_RAFT_ID; + + // init life cycle + + // init server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId); + pSyncNode->pRaftStore = raftStoreOpen(pSyncInfo->walPath); + assert(pSyncNode->pRaftStore != NULL); + + // init candidate vars + pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode); + assert(pSyncNode->pVotesGranted != NULL); + pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode); + assert(pSyncNode->pVotesRespond != NULL); + + // init leader vars + pSyncNode->pNextIndex = NULL; + pSyncNode->pMatchIndex = NULL; // init ping timer pSyncNode->pPingTimer = NULL; @@ -119,6 +153,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer; pSyncNode->heartbeatTimerCounter = 0; + // init callback pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; @@ -135,6 +170,134 @@ void syncNodeClose(SSyncNode* pSyncNode) { free(pSyncNode); } +cJSON* syncNode2Json(const SSyncNode* pSyncNode) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + // init by SSyncInfo + cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + cJSON_AddStringToObject(pRoot, "walPath", pSyncNode->walPath); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient); + cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpSendMsg); + cJSON_AddStringToObject(pRoot, "FpSendMsg", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->queue); + cJSON_AddStringToObject(pRoot, "queue", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpEqMsg); + cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf); + + // init internal + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); + cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); + + cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); + cJSON* pPeers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); + } + cJSON* pPeersId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersId", pPeersId); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + } + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); + cJSON* pReplicasId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + } + + // raft algorithm + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); + cJSON_AddStringToObject(pRoot, "pFsm", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); + cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + + // tla+ server vars + cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); + cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); + + // tla+ candidate vars + + // tla+ leader vars + + // tla+ log vars + + // ping timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); + cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClock); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer); + cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter); + cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); + + // elect timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); + cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClock); + cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer); + cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter); + cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); + + // heartbeat timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClock); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter); + cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); + + // callback + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); + cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); + cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); + cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); + cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); + cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); + return pJson; +} + +char* syncNode2Str(const SSyncNode* pSyncNode) { + cJSON* pJson = syncNode2Json(pSyncNode); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); @@ -183,7 +346,7 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { SRaftId destId; syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); ret = syncNodePing(pSyncNode, &destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); @@ -194,8 +357,8 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId); + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); ret = syncNodePing(pSyncNode, &destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); @@ -204,7 +367,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t syncNodePingSelf(SSyncNode* pSyncNode) { int32_t ret; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &pSyncNode->raftId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); @@ -285,7 +448,7 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { cJSON_Delete(pJson); } - SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->raftId, &pMsg->srcId); + SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsgReply, &rpcMsg); syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); @@ -351,7 +514,38 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { } } -static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {} +static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { + SSyncNode* pSyncNode = (SSyncNode*)param; + if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <= + atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) { + SyncTimeout* pSyncMsg = + syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), + pSyncNode->heartbeatTimerMS, pSyncNode); + + SRpcMsg rpcMsg; + syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + syncTimeoutDestroy(pSyncMsg); + + // reset timer ms + // pSyncNode->heartbeatTimerMS += 100; + + taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, &gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); + } else { + sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", + pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); + } +} + +static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { + if (term > pSyncNode->pRaftStore->currentTerm) { + pSyncNode->pRaftStore->currentTerm = term; + pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID; + raftStorePersist(pSyncNode->pRaftStore); + syncNodeBecomeFollower(pSyncNode); + } +} static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -383,7 +577,7 @@ static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { // static void syncNodeBecomeLeader(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_LEADER; - pSyncNode->leaderCache = pSyncNode->raftId; + pSyncNode->leaderCache = pSyncNode->myRaftId; // next Index +=1 // match Index = 0; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 37e8959ff3..dfbe5db0ed 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -41,7 +41,16 @@ // mdest |-> j]) // /\ UNCHANGED <> // -int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {} +int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + return ret; +} + +int32_t syncNodeReplicate(SSyncNode* pSyncNode) { + // start replicate + int32_t ret = syncNodeAppendEntriesPeers(pSyncNode); + return ret; +} int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) { sTrace("syncNodeAppendEntries pSyncNode:%p ", pSyncNode); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index df9b9d27b4..7cbfd6d40a 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -44,7 +44,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { } else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) { if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->heartbeatTimerCounter); - syncNodeAppendEntriesPeers(ths); + syncNodeReplicate(ths); } } else { sTrace("unknown timeoutType:%d", pMsg->timeoutType); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index c70e490025..216dccd62c 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -95,6 +95,57 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { // ---- misc ---- -int32_t syncUtilRand(int32_t max) { return rand() % max; } +int32_t syncUtilRand(int32_t max) { return taosRand() % max; } -int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } \ No newline at end of file +int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } + +int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); + cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +cJSON* syncUtilRaftId2Json(const SRaftId* p) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + snprintf(u64buf, sizeof(u64buf), "%lu", p->addr); + cJSON_AddStringToObject(pRoot, "addr", u64buf); + char host[128]; + uint16_t port; + syncUtilU642Addr(p->addr, host, sizeof(host), &port); + cJSON_AddStringToObject(pRoot, "host", host); + cJSON_AddNumberToObject(pRoot, "port", port); + cJSON_AddNumberToObject(pRoot, "vgId", p->vgId); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftId", pRoot); + return pJson; +} + +char* syncUtilRaftId2Str(const SRaftId* p) { + cJSON* pJson = syncUtilRaftId2Json(p); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +const char* syncUtilState2String(ESyncState state) { + if (state == TAOS_SYNC_STATE_FOLLOWER) { + return "TAOS_SYNC_STATE_FOLLOWER"; + } else if (state == TAOS_SYNC_STATE_CANDIDATE) { + return "TAOS_SYNC_STATE_CANDIDATE"; + } else if (state == TAOS_SYNC_STATE_LEADER) { + return "TAOS_SYNC_STATE_LEADER"; + } else { + return "TAOS_SYNC_STATE_UNKNOWN"; + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index c9f0ceab57..a2f10ce339 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -14,15 +14,25 @@ */ #include "syncVoteMgr.h" +#include "syncUtil.h" + +// SVotesGranted ----------------------------- +static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { + memset(pVotesGranted->isGranted, 0, sizeof(pVotesGranted->isGranted)); + pVotesGranted->votes = 0; +} SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted)); assert(pVotesGranted != NULL); memset(pVotesGranted, 0, sizeof(SVotesGranted)); - pVotesGranted->quorum = pSyncNode->quorum; + pVotesGranted->replicas = &(pSyncNode->replicasId); + pVotesGranted->replicaNum = pSyncNode->replicaNum; + voteGrantedClearVotes(pVotesGranted); + pVotesGranted->term = 0; - pVotesGranted->votes = 0; + pVotesGranted->quorum = pSyncNode->quorum; pVotesGranted->toLeader = false; pVotesGranted->pSyncNode = pSyncNode; @@ -43,15 +53,73 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) { void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { assert(pMsg->voteGranted == true); assert(pMsg->term == pVotesGranted->term); - pVotesGranted->votes++; + assert(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + + int j = -1; + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + if (syncUtilSameId(&((*(pVotesGranted->replicas))[i]), &(pMsg->srcId))) { + j = i; + break; + } + } + assert(j != -1); + assert(j >= 0 && j < pVotesGranted->replicaNum); + + if (pVotesGranted->isGranted[j] != true) { + ++(pVotesGranted->votes); + pVotesGranted->isGranted[j] = true; + } + assert(pVotesGranted->votes <= pVotesGranted->replicaNum); } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->term = term; - pVotesGranted->votes = 0; + voteGrantedClearVotes(pVotesGranted); pVotesGranted->toLeader = false; } +cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + } + int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); + return pJson; +} + +char *voteGranted2Str(SVotesGranted *pVotesGranted) { + cJSON *pJson = voteGranted2Json(pVotesGranted); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// SVotesRespond ----------------------------- SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond)); assert(pVotesRespond != NULL); @@ -65,6 +133,12 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { return pVotesRespond; } +void votesRespondDestory(SVotesRespond *pVotesRespond) { + if (pVotesRespond != NULL) { + free(pVotesRespond); + } +} + bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { bool ret = false; for (int i = 0; i < pVotesRespond->replicaNum; ++i) { @@ -79,8 +153,8 @@ bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg) { assert(pVotesRespond->term == pMsg->term); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { - if (syncUtilSameId(&(*pVotesRespond->replicas)[i], &pMsg->srcId)) { - assert(pVotesRespond->isRespond[i] == false); + if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { + // assert(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } @@ -88,9 +162,52 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p assert(0); } -void Reset(SVotesRespond *pVotesRespond, SyncTerm term) { +void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { pVotesRespond->term = term; + memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond)); + /* + for (int i = 0; i < pVotesRespond->replicaNum; ++i) { + pVotesRespond->isRespond[i] = false; + } + */ +} + +cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { - pVotesRespond->isRespond[i] = false; + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } + int respondNum = 0; + int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); + for (int i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); + return pJson; +} + +char *votesRespond2Str(SVotesRespond *pVotesRespond) { + cJSON *pJson = votesRespond2Json(pVotesRespond); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; } \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 770d1d1bd8..5a5186c7e2 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -10,6 +10,11 @@ add_executable(syncIOSendMsgServerTest "") add_executable(syncRaftStoreTest "") add_executable(syncEnqTest "") add_executable(syncIndexTest "") +add_executable(syncInitTest "") +add_executable(syncUtilTest "") +add_executable(syncVotesGrantedTest "") +add_executable(syncVotesRespondTest "") +add_executable(syncIndexMgrTest "") target_sources(syncTest @@ -60,6 +65,26 @@ target_sources(syncIndexTest PRIVATE "syncIndexTest.cpp" ) +target_sources(syncInitTest + PRIVATE + "syncInitTest.cpp" +) +target_sources(syncUtilTest + PRIVATE + "syncUtilTest.cpp" +) +target_sources(syncVotesGrantedTest + PRIVATE + "syncVotesGrantedTest.cpp" +) +target_sources(syncVotesRespondTest + PRIVATE + "syncVotesRespondTest.cpp" +) +target_sources(syncIndexMgrTest + PRIVATE + "syncIndexMgrTest.cpp" +) target_include_directories(syncTest @@ -122,6 +147,31 @@ target_include_directories(syncIndexTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncInitTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncUtilTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncVotesGrantedTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncVotesRespondTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncIndexMgrTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -172,6 +222,26 @@ target_link_libraries(syncIndexTest sync gtest_main ) +target_link_libraries(syncInitTest + sync + gtest_main +) +target_link_libraries(syncUtilTest + sync + gtest_main +) +target_link_libraries(syncVotesGrantedTest + sync + gtest_main +) +target_link_libraries(syncVotesRespondTest + sync + gtest_main +) +target_link_libraries(syncIndexMgrTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 0bf43f933e..e2bc9a73ae 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -84,7 +84,7 @@ int main(int argc, char** argv) { gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; for (int i = 0; i < 10; ++i) { - SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->raftId, &pSyncNode->raftId); + SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp new file mode 100644 index 0000000000..4e4cd9222b --- /dev/null +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -0,0 +1,141 @@ +#include "syncIndexMgr.h" +//#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + char* serialized = syncNode2Str(pSyncNode); + printf("%s\n", serialized); + free(serialized); + + initRaftId(pSyncNode); + + SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode); + assert(pSyncIndexMgr != NULL); + + printf("---------------------------------------\n"); + { + char* serialized = syncIndexMgr2Str(pSyncIndexMgr); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); + syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200); + syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300); + + printf("---------------------------------------\n"); + { + char* serialized = syncIndexMgr2Str(pSyncIndexMgr); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + printf("---------------------------------------\n"); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); + printf("index %d : %lu \n", i, idx); + } + + syncIndexMgrClear(pSyncIndexMgr); + printf("---------------------------------------\n"); + { + char* serialized = syncIndexMgr2Str(pSyncIndexMgr); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + syncIndexMgrDestroy(pSyncIndexMgr); + return 0; +} diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp new file mode 100644 index 0000000000..669c4e68a5 --- /dev/null +++ b/source/libs/sync/test/syncInitTest.cpp @@ -0,0 +1,99 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + char* serialized = syncNode2Str(pSyncNode); + printf("%s\n", serialized); + free(serialized); + + initRaftId(pSyncNode); + + return 0; +} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 4e5f7b78b5..450e097cc8 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -25,7 +25,8 @@ SSyncNode* doSync(int myIndex) { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./wal_path"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp new file mode 100644 index 0000000000..9a1c113620 --- /dev/null +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -0,0 +1,32 @@ +#include "syncUtil.h" +//#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void electRandomMSTest() { + for (int i = 0; i < 10; ++i) { + int32_t ms = syncUtilElectRandomMS(); + printf("syncUtilElectRandomMS: %d \n", ms); + } +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + logTest(); + electRandomMSTest(); + + return 0; +} diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp new file mode 100644 index 0000000000..3edde509f8 --- /dev/null +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + char* serialized = syncNode2Str(pSyncNode); + printf("%s\n", serialized); + free(serialized); + + initRaftId(pSyncNode); + + SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode); + assert(pVotesGranted != NULL); + + printf("---------------------------------------\n"); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + SyncTerm term = 1234; + printf("---------------------------------------\n"); + voteGrantedReset(pVotesGranted, term); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + for (int i = 0; i < replicaNum; ++i) { + SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild(); + reply->destId = pSyncNode->myRaftId; + reply->srcId = ids[i]; + reply->term = term; + reply->voteGranted = true; + + voteGrantedVote(pVotesGranted, reply); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + voteGrantedVote(pVotesGranted, reply); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + } + + printf("---------------------------------------\n"); + voteGrantedReset(pVotesGranted, 123456789); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + voteGrantedDestroy(pVotesGranted); + return 0; +} diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp new file mode 100644 index 0000000000..74d42cd531 --- /dev/null +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + char* serialized = syncNode2Str(pSyncNode); + printf("%s\n", serialized); + free(serialized); + + initRaftId(pSyncNode); + + SVotesRespond* pVotesRespond = votesRespondCreate(pSyncNode); + assert(pVotesRespond != NULL); + + printf("---------------------------------------\n"); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + SyncTerm term = 1234; + printf("---------------------------------------\n"); + votesRespondReset(pVotesRespond, term); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + for (int i = 0; i < replicaNum; ++i) { + SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild(); + reply->destId = pSyncNode->myRaftId; + reply->srcId = ids[i]; + reply->term = term; + reply->voteGranted = true; + + votesRespondAdd(pVotesRespond, reply); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + votesRespondAdd(pVotesRespond, reply); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + } + + printf("---------------------------------------\n"); + votesRespondReset(pVotesRespond, 123456789); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + votesRespondDestory(pVotesRespond); + return 0; +} diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index fe0f3befd6..237a39e47d 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -27,7 +27,7 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { ((uint64_t *)fileid)[0] = stDev; ((uint64_t *)fileid)[1] = stIno; if (unique) { - ((uint64_t *)fileid)[2] = rand(); + ((uint64_t *)fileid)[2] = taosRand(); } return 0; diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index 913f34d6c2..f16d0445c6 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -59,7 +59,7 @@ typedef struct STfsDir { SDiskID did; char dirname[TSDB_FILENAME_LEN]; STfsFile tfile; - DIR *dir; + TdDirPtr pDir; STfs *pTfs; } STfsDir; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 2579490791..c46989dc5d 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -192,14 +192,14 @@ void tfsBasename(const STfsFile *pFile, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); - tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN); + tstrncpy(dest, taosDirEntryBaseName(tname), TSDB_FILENAME_LEN); } void tfsDirname(const STfsFile *pFile, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); - tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); + tstrncpy(dest, taosDirName(tname), TSDB_FILENAME_LEN); } int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } @@ -233,7 +233,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { // the pointer directly in this recursion. // See // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html - char *dir = strdup(dirname(s)); + char *dir = strdup(taosDirName(s)); if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { free(s); @@ -324,45 +324,46 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { return pDir; } -const STfsFile *tfsReaddir(STfsDir *pDir) { - if (pDir == NULL || pDir->dir == NULL) return NULL; +const STfsFile *tfsReaddir(STfsDir *pTfsDir) { + if (pTfsDir == NULL || pTfsDir->pDir == NULL) return NULL; char bname[TMPNAME_LEN * 2] = "\0"; while (true) { - struct dirent *dp = NULL; - dp = readdir(pDir->dir); - if (dp != NULL) { + TdDirEntryPtr pDirEntry = NULL; + pDirEntry = taosReadDir(pTfsDir->pDir); + if (pDirEntry != NULL) { // Skip . and .. - if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; + char *name = taosGetDirEntryName(pDirEntry); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - if (pDir->dirname == NULL || pDir->dirname[0] == 0) { - snprintf(bname, TMPNAME_LEN * 2, "%s", dp->d_name); + if (pTfsDir->dirname == NULL || pTfsDir->dirname[0] == 0) { + snprintf(bname, TMPNAME_LEN * 2, "%s", name); } else { - snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); + snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pTfsDir->dirname, TD_DIRSEP, name); } - tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname); - return &pDir->tfile; + tfsInitFile(pTfsDir->pTfs, &pTfsDir->tfile, pTfsDir->did, bname); + return &pTfsDir->tfile; } - if (tfsOpendirImpl(pDir->pTfs, pDir) < 0) { + if (tfsOpendirImpl(pTfsDir->pTfs, pTfsDir) < 0) { return NULL; } - if (pDir->dir == NULL) { + if (pTfsDir->pDir == NULL) { terrno = TSDB_CODE_SUCCESS; return NULL; } } } -void tfsClosedir(STfsDir *pDir) { - if (pDir) { - if (pDir->dir != NULL) { - closedir(pDir->dir); - pDir->dir = NULL; +void tfsClosedir(STfsDir *pTfsDir) { + if (pTfsDir) { + if (pTfsDir->pDir != NULL) { + taosCloseDir(pTfsDir->pDir); + pTfsDir->pDir = NULL; } - free(pDir); + free(pTfsDir); } } @@ -487,29 +488,29 @@ static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) { return pDisk; } -static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) { +static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { STfsDisk *pDisk = NULL; char adir[TMPNAME_LEN * 2] = "\0"; - if (pDir->dir != NULL) { - closedir(pDir->dir); - pDir->dir = NULL; + if (pTfsDir->pDir != NULL) { + taosCloseDir(pTfsDir->pDir); + pTfsDir->pDir = NULL; } while (true) { - pDisk = tfsNextDisk(pTfs, &pDir->iter); + pDisk = tfsNextDisk(pTfs, &pTfsDir->iter); if (pDisk == NULL) return 0; - pDir->did.level = pDisk->level; - pDir->did.id = pDisk->id; + pTfsDir->did.level = pDisk->level; + pTfsDir->did.id = pDisk->id; if (pDisk->path == NULL || pDisk->path[0] == 0) { - snprintf(adir, TMPNAME_LEN * 2, "%s", pDir->dirname); + snprintf(adir, TMPNAME_LEN * 2, "%s", pTfsDir->dirname); } else { - snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pTfsDir->dirname); } - pDir->dir = opendir(adir); - if (pDir->dir != NULL) break; + pTfsDir->pDir = taosOpenDir(adir); + if (pTfsDir->pDir != NULL) break; } return 0; diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 465646ac95..5cc436cf32 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common + PUBLIC zlib ) if (${BUILD_WITH_UV_TRANS}) if (${BUILD_WITH_UV}) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index e1319da162..615c576a9b 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -749,7 +749,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { memcpy(pConn->user, pHead->user, tListLen(pConn->user)); pConn->pRpc = pRpc; pConn->sid = sid; - pConn->tranId = (uint16_t)(rand() & 0xFFFF); + pConn->tranId = (uint16_t)(taosRand() & 0xFFFF); pConn->ownId = htonl(pConn->sid); pConn->linkUid = pHead->linkUid; if (pRpc->afp) { diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index d95ac3d36d..aac38b21e8 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -24,7 +24,7 @@ #ifndef USE_UV typedef struct SFdObj { void * signature; - SOCKET fd; // TCP socket FD + TdSocketPtr pSocket; // TCP socket FD void * thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; @@ -40,7 +40,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - EpollFd pollFd; + TdEpollPtr pEpoll; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -56,20 +56,20 @@ typedef struct { } SClientObj; typedef struct { - SOCKET fd; - uint32_t ip; - uint16_t port; - int8_t stop; - int8_t reserve; - char label[TSDB_LABEL_LEN]; - int numOfThreads; - void * shandle; - SThreadObj **pThreadObj; - pthread_t thread; + TdSocketServerPtr pSocketServer; + uint32_t ip; + uint16_t port; + int8_t stop; + int8_t reserve; + char label[TSDB_LABEL_LEN]; + int numOfThreads; + void * shandle; + SThreadObj **pThreadObj; + pthread_t thread; } SServerObj; static void * taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void * taosAcceptTcpConnection(void *arg); @@ -85,7 +85,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return NULL; } - pServerObj->fd = -1; + pServerObj->pSocketServer = NULL; taosResetPthread(&pServerObj->thread); pServerObj->ip = ip; pServerObj->port = port; @@ -118,7 +118,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pServerObj->pThreadObj[i] = pThreadObj; - pThreadObj->pollFd = -1; + pThreadObj->pEpoll = NULL; taosResetPthread(&pThreadObj->thread); pThreadObj->processData = fp; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); @@ -135,8 +135,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter - if (pThreadObj->pollFd < 0) { + pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter + if (pThreadObj->pEpoll == NULL) { tError("%s failed to create TCP epoll", label); code = -1; break; @@ -151,8 +151,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pThreadObj->threadId = i; } - pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); - if (pServerObj->fd < 0) code = -1; + pServerObj->pSocketServer = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + if (pServerObj->pSocketServer == NULL) code = -1; if (code == 0) { code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); @@ -196,8 +196,8 @@ void taosStopTcpServer(void *handle) { if (pServerObj == NULL) return; pServerObj->stop = 1; - if (pServerObj->fd >= 0) { - taosShutDownSocketRD(pServerObj->fd); + if (pServerObj->pSocketServer != NULL) { + taosShutDownSocketServerRD(pServerObj->pSocketServer); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -227,7 +227,7 @@ void taosCleanUpTcpServer(void *handle) { } static void *taosAcceptTcpConnection(void *arg) { - SOCKET connFd = -1; + TdSocketPtr pSocket = NULL; struct sockaddr_in caddr; int threadId = 0; SThreadObj * pThreadObj; @@ -239,13 +239,13 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); - connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + pSocket = taosAcceptTcpConnectSocket(pServerObj->pSocketServer, (struct sockaddr *)&caddr, &addrlen); if (pServerObj->stop) { tDebug("%s TCP server stop accepting new connections", pServerObj->label); break; } - if (connFd == -1) { + if (pSocket == NULL) { if (errno == EINVAL) { tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); break; @@ -255,11 +255,11 @@ static void *taosAcceptTcpConnection(void *arg) { continue; } - taosKeepTcpAlive(connFd); + taosKeepTcpAlive(pSocket); struct timeval to = {5, 0}; - int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + int32_t ret = taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); continue; @@ -268,14 +268,14 @@ static void *taosAcceptTcpConnection(void *arg) { // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; - SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket); if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + tDebug("%s new TCP connection from %s:%hu, FD:%p numOfFds:%d", pServerObj->label, + taosInetNtoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); } else { - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); } @@ -285,7 +285,7 @@ static void *taosAcceptTcpConnection(void *arg) { threadId = threadId % pServerObj->numOfThreads; } - taosCloseSocket(pServerObj->fd); + taosCloseSocketServer(&pServerObj->pSocketServer); return NULL; } @@ -339,8 +339,8 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter - if (pThreadObj->pollFd < 0) { + pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter + if (pThreadObj->pEpoll == NULL) { tError("%s failed to create TCP epoll", label); code = -1; break; @@ -388,21 +388,17 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin atomic_store_32(&pClientObj->index, index + 1); SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; - SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd == (SOCKET)-1) return NULL; -#else - if (fd <= 0) return NULL; -#endif + TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); + if (pSocket == NULL) return NULL; struct sockaddr_in sin; uint16_t localPort = 0; unsigned int addrlen = sizeof(sin); - if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + if (taosGetSocketName(pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { localPort = (uint16_t)ntohs(sin.sin_port); } - SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket); if (pFdObj) { pFdObj->thandle = thandle; @@ -415,7 +411,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin ipport, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); - taosCloseSocket(fd); + taosCloseSocket(&pSocket); } return pFdObj; @@ -430,7 +426,7 @@ void taosCloseTcpConnection(void *chandle) { // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; - taosShutDownSocketWR(pFdObj->fd); + taosShutDownSocketWR(pFdObj->pSocket); } int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { @@ -438,8 +434,8 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1; SThreadObj *pThreadObj = pFdObj->pThreadObj; - int ret = taosWriteMsg(pFdObj->fd, data, len); - tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + int ret = taosWriteMsg(pFdObj->pSocket, data, len); + tTrace("%s %p TCP data is sent, FD:%p bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, ret); return ret; } @@ -449,7 +445,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { // notify the upper layer, so it will clean the associated context if (pFdObj->closedByApp == 0) { - taosShutDownSocketWR(pFdObj->fd); + taosShutDownSocketWR(pFdObj->pSocket); SRecvInfo recvInfo; recvInfo.msg = NULL; @@ -473,7 +469,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { SThreadObj *pThreadObj = pFdObj->pThreadObj; - headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); + headLen = taosReadMsg(pFdObj->pSocket, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); return -1; @@ -486,13 +482,12 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, - buffer); + tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer); } msg = buffer + tsRpcOverhead; leftLen = msgLen - headLen; - retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); + retLen = taosReadMsg(pFdObj->pSocket, msg + headLen, leftLen); if (leftLen != retLen) { tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); @@ -532,7 +527,7 @@ static void *taosProcessTcpData(void *param) { setThreadName(name); while (1) { - int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); + int fdNum = taosWaitEpoll(pThreadObj->pEpoll, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { tDebug("%s TCP thread get stop event, exiting...", pThreadObj->label); break; @@ -561,7 +556,7 @@ static void *taosProcessTcpData(void *param) { } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - shutdown(pFdObj->fd, SHUT_WR); + taosShutDownSocketWR(pFdObj->pSocket); continue; } @@ -572,9 +567,9 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >= 0) { - EpollClose(pThreadObj->pollFd); - pThreadObj->pollFd = -1; + if (pThreadObj->pEpoll != NULL) { + taosCloseEpoll(&pThreadObj->pEpoll); + pThreadObj->pEpoll = NULL; } while (pThreadObj->pHead) { @@ -590,7 +585,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); @@ -599,13 +594,13 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { } pFdObj->closedByApp = 0; - pFdObj->fd = fd; + pFdObj->pSocket = pSocket; pFdObj->pThreadObj = pThreadObj; pFdObj->signature = pFdObj; event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { + if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) { tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -635,8 +630,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { } pFdObj->signature = NULL; - epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - taosCloseSocket(pFdObj->fd); + taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_DEL, pFdObj->pSocket, NULL); + taosCloseSocket(&pFdObj->pSocket); pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) @@ -655,8 +650,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, - pFdObj->fd, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); tfree(pFdObj); } diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 3640414a4c..81c8d0af76 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -30,17 +30,17 @@ #define RPC_MAX_UDP_SIZE 65480 typedef struct { - int index; - SOCKET fd; - uint16_t port; // peer port - uint16_t localPort; // local port - char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; - void * hash; - void * shandle; // handle passed by upper layer during server initialization - void * pSet; - void *(*processData)(SRecvInfo *pRecv); - char *buffer; // buffer to receive data + int index; + TdSocketPtr pSocket; + uint16_t port; // peer port + uint16_t localPort; // local port + char label[TSDB_LABEL_LEN]; // copy from udpConnSet; + pthread_t thread; + void *hash; + void *shandle; // handle passed by upper layer during server initialization + void *pSet; + void *(*processData)(SRecvInfo *pRecv); + char *buffer; // buffer to receive data } SUdpConn; typedef struct { @@ -86,8 +86,8 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads for (i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; ownPort = (port ? port + i : 0); - pConn->fd = taosOpenUdpSocket(ip, ownPort); - if (pConn->fd < 0) { + pConn->pSocket = taosOpenUdpSocket(ip, ownPort); + if (pConn->pSocket == NULL) { tError("%s failed to open UDP socket %x:%hu", label, ip, port); break; } @@ -100,7 +100,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads struct sockaddr_in sin; unsigned int addrlen = sizeof(sin); - if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && + if (taosGetSocketName(pConn->pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { pConn->localPort = (uint16_t)ntohs(sin.sin_port); } @@ -138,9 +138,9 @@ void taosStopUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR); - if (pConn->fd >= 0) taosCloseSocket(pConn->fd); - pConn->fd = -1; + if (pConn->pSocket != NULL) taosShutDownSocketRDWR(pConn->pSocket); + if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket); + pConn->pSocket = NULL; } for (int i = 0; i < pSet->threads; ++i) { @@ -163,7 +163,7 @@ void taosCleanUpUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >= 0) taosCloseSocket(pConn->fd); + if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket); } tDebug("%s UDP is cleaned up", pSet->label); @@ -199,13 +199,12 @@ static void *taosRecvUdpData(void *param) { setThreadName("recvUdpData"); while (1) { - dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); + dataLen = taosReadFromSocket(pConn->pSocket, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); if (dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen, - pConn->fd); + tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d", pConn->label, strerror(errno), (int32_t)dataLen); // for windows usage, remote shutdown also returns - 1 in windows client - if (pConn->fd == -1) { + if (pConn->pSocket == NULL) { break; } else { continue; @@ -255,7 +254,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c destAdd.sin_addr.s_addr = ip; destAdd.sin_port = htons(port); - int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); + int ret = taosSendto(pConn->pSocket, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); return ret; } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 95e67290ac..6d1c691b9b 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -173,7 +173,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 #else int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { int32_t code = -1; - SOCKET fd = 0; + TdSocketPtr pSocket = NULL; uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { @@ -182,8 +182,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 goto SEND_OVER; } - fd = taosOpenTcpClientSocket(ip, port, 0); - if (fd < 0) { + pSocket = taosOpenTcpClientSocket(ip, port, 0); + if (pSocket == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create http socket to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; @@ -200,21 +200,20 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 char header[1024] = {0}; int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); - - if (taosWriteSocket(fd, header, headLen) < 0) { + if (taosWriteMsg(pSocket, header, headLen) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to send http header to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } - if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + if (taosWriteMsg(pSocket, (void*)pCont, contLen) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to send http content to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } // read something to avoid nginx error 499 - if (taosReadSocket(fd, header, 10) < 0) { + if (taosWriteMsg(pSocket, header, 10) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to receive response from %s:%u since %s", server, port, terrstr()); goto SEND_OVER; @@ -223,8 +222,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 code = 0; SEND_OVER: - if (fd != 0) { - taosCloseSocket(fd); + if (pSocket != NULL) { + taosCloseSocket(&pSocket); } return code; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2a4f3497e4..248f758787 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -130,16 +130,16 @@ int walCheckAndRepairMeta(SWal* pWal) { regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); - DIR* dir = opendir(pWal->path); - if (dir == NULL) { + TdDirPtr pDir = taosOpenDir(pWal->path); + if (pDir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } // scan log files and build new meta - struct dirent* ent; - while ((ent = readdir(dir)) != NULL) { - char* name = basename(ent->d_name); + TdDirEntryPtr pDirEntry; + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry)); int code = regexec(&logRegPattern, name, 0, NULL, 0); if (code == 0) { SWalFileInfo fileInfo; @@ -149,7 +149,7 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - closedir(dir); + taosCloseDir(pDir); regfree(&logRegPattern); regfree(&idxRegPattern); @@ -337,25 +337,25 @@ static int walFindCurMetaVer(SWal* pWal) { regex_t walMetaRegexPattern; regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED); - DIR* dir = opendir(pWal->path); - if (dir == NULL) { + TdDirPtr pDir = taosOpenDir(pWal->path); + if (pDir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } - struct dirent* ent; + TdDirEntryPtr pDirEntry; // find existing meta-ver[x].json int metaVer = -1; - while ((ent = readdir(dir)) != NULL) { - char* name = basename(ent->d_name); + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry)); int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); if (code == 0) { sscanf(name, "meta-ver%d", &metaVer); break; } } - closedir(dir); + taosCloseDir(pDir); regfree(&walMetaRegexPattern); return metaVer; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 699da75790..0f6ac0b214 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -34,9 +34,6 @@ int32_t walCommit(SWal *pWal, int64_t ver) { int32_t walRollback(SWal *pWal, int64_t ver) { int code; char fnameStr[WAL_FILE_LEN]; - if (ver == pWal->vers.lastVer) { - return 0; - } if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 230555e016..f44fc71964 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -124,13 +124,8 @@ class WalRetentionEnv : public ::testing::Test { void SetUp() override { SWalCfg cfg; - cfg.rollPeriod = -1, - cfg.segSize = -1, - cfg.retentionPeriod = -1, - cfg.retentionSize = 0, - cfg.rollPeriod = 0, - cfg.vgId = 0, - cfg.level = TAOS_WAL_FSYNC; + cfg.rollPeriod = -1, cfg.segSize = -1, cfg.retentionPeriod = -1, cfg.retentionSize = 0, cfg.rollPeriod = 0, + cfg.vgId = 0, cfg.level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, &cfg); ASSERT(pWal != NULL); } @@ -241,6 +236,12 @@ TEST_F(WalCleanEnv, rollback) { ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); code = walRollback(pWal, 5); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, 4); @@ -299,7 +300,7 @@ TEST_F(WalKeepEnv, readHandleRead) { ASSERT_EQ(code, 0); } for (int i = 0; i < 1000; i++) { - int ver = rand() % 100; + int ver = taosRand() % 100; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); @@ -324,7 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) { TEST_F(WalRetentionEnv, repairMeta1) { walResetEnv(); int code; - + int i; for (i = 0; i < 100; i++) { char newStr[100]; @@ -336,14 +337,14 @@ TEST_F(WalRetentionEnv, repairMeta1) { TearDown(); - //getchar(); + // getchar(); char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); taosRemoveFile(buf); sprintf(buf, "%s/meta-ver%d", pathName, 1); taosRemoveFile(buf); SetUp(); - //getchar(); + // getchar(); ASSERT_EQ(pWal->vers.lastVer, 99); @@ -351,7 +352,7 @@ TEST_F(WalRetentionEnv, repairMeta1) { ASSERT(pRead != NULL); for (int i = 0; i < 1000; i++) { - int ver = rand() % 100; + int ver = taosRand() % 100; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); @@ -381,7 +382,7 @@ TEST_F(WalRetentionEnv, repairMeta1) { } for (int i = 0; i < 1000; i++) { - int ver = rand() % 200; + int ver = taosRand() % 200; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); @@ -401,5 +402,4 @@ TEST_F(WalRetentionEnv, repairMeta1) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } } - } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 7d7382d83f..b4058b3c0e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #include "osString.h" @@ -36,6 +37,10 @@ #include #include +typedef struct dirent dirent; +typedef struct DIR TdDir; +typedef struct dirent TdDirent; + void taosRemoveDir(const char *dirname) { DIR *dir = opendir(dirname); if (dir == NULL) return; @@ -149,4 +154,47 @@ bool taosIsDir(const char *dirname) { return false; } +char* taosDirName(char *name) { + return dirname(name); +} + +char* taosDirEntryBaseName(char *name) { + return basename(name); +} + +TdDirPtr taosOpenDir(const char *dirname) { + if (dirname == NULL) { + return NULL; + } + return (TdDirPtr)opendir(dirname); +} + +TdDirEntryPtr taosReadDir(TdDirPtr pDir) { + if (pDir == NULL) { + return NULL; + } + return (TdDirEntryPtr)readdir((DIR*)pDir); +} + +bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { + if (pDirEntry == NULL) { + return false; + } + return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0; +} + +char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) { + if (pDirEntry == NULL) { + return NULL; + } + return ((dirent*)pDirEntry)->d_name; +} + +int32_t taosCloseDir(TdDirPtr pDir) { + if (pDir == NULL) { + return -1; + } + return closedir((DIR*)pDir); +} + #endif diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 0b2fe904b3..63fa600217 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -38,7 +38,7 @@ float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; void osInit() { - srand(taosSafeRand()); + taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); taosGetSystemTimezone(tsTimezone); taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index b81e41b3cf..f3dd9b74c5 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -21,8 +21,12 @@ #include #endif +void taosSeedRand(uint32_t seed) { return srand(seed); } + uint32_t taosRand(void) { return rand(); } +uint32_t taosRandR(uint32_t *pSeed) { return rand_r(pSeed); } + uint32_t taosSafeRand(void) { TdFilePtr pFile; int seed; diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 0d7066b5c8..e5b506e8d8 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -99,7 +99,7 @@ static void *sem_thread_routine(void *arg) { sem_port = mach_task_self(); kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); if (ret != KERN_SUCCESS) { - fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); sem_inited = -1; return NULL; } @@ -112,7 +112,7 @@ static void once_init(void) { int r = 0; r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); return; } while (sem_inited == 0) { @@ -139,14 +139,14 @@ struct tsem_s { }; int tsem_init(tsem_t *sem, int pshared, unsigned int value) { - // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); if (*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p)); if (!p) { - fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -162,7 +162,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { p->val = value; } while (0); if (r) { - fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #elif defined(SEM_USE_POSIX) @@ -181,27 +181,27 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { int e = errno; if (e == EEXIST) continue; if (e == EINTR) continue; - fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e, strerror(e)); abort(); } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) pthread_once(&sem_once, once_init); if (sem_inited != 1) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); errno = ENOMEM; return -1; } kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); if (ret != KERN_SUCCESS) { - fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // we fail-fast here, because we have less-doc about semaphore_create for the moment abort(); } #else // SEM_USE_PTHREAD p->sem = dispatch_semaphore_create(value); if (p->sem == NULL) { - fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #endif // SEM_USE_PTHREAD @@ -215,28 +215,28 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { int tsem_wait(tsem_t *sem) { if (!*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } struct tsem_s *p = *sem; if (!p->valid) { - fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #ifdef SEM_USE_PTHREAD if (pthread_mutex_lock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val -= 1; if (p->val < 0) { if (pthread_cond_wait(&p->cond, &p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } if (pthread_mutex_unlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } return 0; @@ -251,28 +251,28 @@ int tsem_wait(tsem_t *sem) { int tsem_post(tsem_t *sem) { if (!*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } struct tsem_s *p = *sem; if (!p->valid) { - fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #ifdef SEM_USE_PTHREAD if (pthread_mutex_lock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val += 1; if (p->val <= 0) { if (pthread_cond_signal(&p->cond)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } if (pthread_mutex_unlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } return 0; @@ -286,34 +286,34 @@ int tsem_post(tsem_t *sem) { } int tsem_destroy(tsem_t *sem) { - // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); if (!*sem) { - // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // abort(); return 0; } struct tsem_s *p = *sem; if (!p->valid) { - // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // abort(); return 0; } #ifdef SEM_USE_PTHREAD if (pthread_mutex_lock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->valid = 0; if (pthread_cond_destroy(&p->cond)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } if (pthread_mutex_unlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } if (pthread_mutex_destroy(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #elif defined(SEM_USE_POSIX) @@ -322,7 +322,7 @@ int tsem_destroy(tsem_t *sem) { int r = sem_unlink(name); if (r) { int e = errno; - fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e, strerror(e)); abort(); } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 096516a98d..698ceded16 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -18,176 +18,206 @@ #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #include "winsock2.h" - #include - #include - #include -#else - #include - #include - #include - #include - #include - #include - #include - #include - #include -#endif - -// typedef struct TdSocketServer { -// #if SOCKET_WITH_LOCK -// pthread_rwlock_t rwlock; -// #endif -// int refId; -// SocketFd fd; -// } * TdSocketServerPtr, TdSocketServer; - -// typedef struct TdSocketConnector { -// #if SOCKET_WITH_LOCK -// pthread_rwlock_t rwlock; -// #endif -// int refId; -// SocketFd fd; -// } * TdSocketConnectorPtr, TdSocketConnector; - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -#define taosSend(sockfd, buf, len, flags) send((SOCKET)sockfd, buf, len, flags) -int32_t taosSendto(SocketFd fd, void *buf, int len, unsigned int flags, const struct sockaddr *to, int tolen) { - return sendto((SOCKET)sockfd, buf, len, flags, dest_addr, addrlen); -} -int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { return send((SOCKET)fd, buf, len, 0); } -int32_t taosReadSocket(SocketFd fd, void *buf, int len) { return recv((SOCKET)fd, buf, len, 0)(); } -int32_t taosCloseSocketNoCheck(SocketFd fd) { return closesocket((SOCKET)fd); } -int32_t taosCloseSocket(SocketFd fd) { closesocket((SOCKET)fd) } - -#else - - #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) - int32_t taosSendto(SocketFd fd, void * buf, int len, unsigned int flags, const struct sockaddr * dest_addr, int addrlen) { - return sendto(fd, buf, len, flags, dest_addr, addrlen); - } - - int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { - return write(fd, buf, len); - } - - int32_t taosReadSocket(SocketFd fd, void *buf, int len) { - return read(fd, buf, len); - } - - int32_t taosCloseSocketNoCheck(SocketFd fd) { - return close(fd); - } - - int32_t taosCloseSocket(SocketFd fd) { - if (fd > -1) { - close(fd); - } - } -#endif - -void taosShutDownSocketRD(SOCKET fd) { -#ifdef WINDOWS - closesocket(fd); -#elif __APPLE__ - close(fd); -#else - shutdown(fd, SHUT_RD); -#endif -} - -void taosShutDownSocketWR(SOCKET fd) { -#ifdef WINDOWS - closesocket(fd); -#elif __APPLE__ - close(fd); -#else - shutdown(fd, SHUT_WR); -#endif -} - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) - -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { - int32_t flags = 0; - if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { - //printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - if (on) - flags |= O_NONBLOCK; - else - flags &= ~O_NONBLOCK; - - if ((flags = fcntl(sock, F_SETFL, flags)) < 0) { - //printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - return 0; -} - - - - -#endif - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_32)) - -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} - -int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen) { - return getsockopt(socketfd, level, optname, optval, (socklen_t *)optlen); -} - -#endif - -#if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) - -uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } - -const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } - -#endif - -#if defined(_TD_DARWIN_64) - -/* - * darwin implementation - */ - -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { - if (level == SOL_SOCKET && optname == SO_SNDBUF) { - return 0; - } - - if (level == SOL_SOCKET && optname == SO_RCVBUF) { - return 0; - } - - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} -#endif - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -/* - * windows implementation - */ - #include #include +#include #include #include #include +#include #include #include +#include "winsock2.h" +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif -void taosWinSocketInit() { +typedef int32_t SocketFd; +typedef SocketFd EpollFd; + +typedef struct TdSocketServer { +#if SOCKET_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketServerPtr, TdSocketServer; + +typedef struct TdSocket { +#if SOCKET_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketPtr, TdSocket; + +typedef struct TdEpoll { +#if SOCKET_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + EpollFd fd; +} *TdEpollPtr, TdEpoll; + +int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, const struct sockaddr *dest_addr, + int addrlen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); +#else + return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); +#endif +} +int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return send(pSocket->fd, buf, len, 0); + ; +#else + return write(pSocket->fd, buf, len); +#endif +} +int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return recv(pSocket->fd, buf, len, 0); + ; +#else + return read(pSocket->fd, buf, len); +#endif +} + +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } + return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); +} +int32_t taosCloseSocketNoCheck1(SocketFd fd) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return closesocket(fd); +#else + return close(fd); +#endif +} +int32_t taosCloseSocket(TdSocketPtr *ppSocket) { + int32_t code; + if (ppSocket == NULL || *ppSocket == NULL || (*ppSocket)->fd < 0) { + return -1; + } + code = taosCloseSocketNoCheck1((*ppSocket)->fd); + (*ppSocket)->fd = -1; + free(*ppSocket); + return code; +} +int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { + int32_t code; + if (ppSocketServer == NULL || *ppSocketServer == NULL || (*ppSocketServer)->fd < 0) { + return -1; + } + code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); + (*ppSocketServer)->fd = -1; + free(*ppSocketServer); + return code; +} + +int32_t taosShutDownSocketRD(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocket->fd); +#elif __APPLE__ + return close(pSocket->fd); +#else + return shutdown(pSocket->fd, SHUT_RD); +#endif +} +int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer) { + if (pSocketServer == NULL || pSocketServer->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocketServer->fd); +#elif __APPLE__ + return close(pSocketServer->fd); +#else + return shutdown(pSocketServer->fd, SHUT_RD); +#endif +} + +int32_t taosShutDownSocketWR(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocket->fd); +#elif __APPLE__ + return close(pSocket->fd); +#else + return shutdown(pSocket->fd, SHUT_WR); +#endif +} +int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer) { + if (pSocketServer == NULL || pSocketServer->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocketServer->fd); +#elif __APPLE__ + return close(pSocketServer->fd); +#else + return shutdown(pSocketServer->fd, SHUT_WR); +#endif +} +int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocket->fd); +#elif __APPLE__ + return close(pSocket->fd); +#else + return shutdown(pSocket->fd, SHUT_RDWR); +#endif +} +int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { + if (pSocketServer == NULL || pSocketServer->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocketServer->fd); +#elif __APPLE__ + return close(pSocketServer->fd); +#else + return shutdown(pSocketServer->fd, SHUT_RDWR); +#endif +} + +#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) + #if defined(_TD_GO_DLL_) + uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } + #endif +#endif + +void taosWinSocketInit1() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) static char flag = 0; if (flag == 0) { WORD wVersionRequested; @@ -197,21 +227,46 @@ void taosWinSocketInit() { flag = 1; } } +#else +#endif } - -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { +int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) u_long mode; if (on) { mode = 1; - ioctlsocket(sock, FIONBIO, &mode); + ioctlsocket(pSocket->fd, FIONBIO, &mode); } else { mode = 0; - ioctlsocket(sock, FIONBIO, &mode); + ioctlsocket(pSocket->fd, FIONBIO, &mode); } +#else + int32_t flags = 0; + if ((flags = fcntl(pSocket->fd, F_GETFL, 0)) < 0) { + // printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); + return 1; + } + + if (on) + flags |= O_NONBLOCK; + else + flags &= ~O_NONBLOCK; + + if ((flags = fcntl(pSocket->fd, F_SETFL, flags)) < 0) { + // printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); + return 1; + } +#endif return 0; } - -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -228,13 +283,23 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op return 0; } - return setsockopt(socketfd, level, optname, optval, optlen); + return setsockopt(pSocket->fd, level, optname, optval, optlen); +#else + return setsockopt(pSocket->fd, level, optname, optval, (socklen_t)optlen); +#endif +} +int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return getsockopt(pSocket->fd, level, optname, optval, (socklen_t *)optlen); +#endif } - -#ifdef _MSC_VER -//#if _MSC_VER >= 1900 - uint32_t taosInetAddr(const char *ipAddr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { @@ -242,39 +307,37 @@ uint32_t taosInetAddr(const char *ipAddr) { } else { return value; } +#else + return inet_addr(ipAddr); +#endif } - const char *taosInetNtoa(struct in_addr ipInt) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); +#else + return inet_ntoa(ipInt); +#endif } -//#endif -#endif - -#if defined(_TD_GO_DLL_) - -uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } - -#endif - -#endif - #ifndef SIGPIPE - #define SIGPIPE EPIPE +#define SIGPIPE EPIPE #endif #define TCP_CONN_TIMEOUT 3000 // conn timeout -int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } int32_t nleft, nwritten; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; while (nleft > 0) { - nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); + nwritten = taosWriteSocket(pSocket, (char *)ptr, (size_t)nleft); if (nwritten <= 0) { if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) continue; @@ -293,20 +356,21 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } int32_t nleft, nread; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; - if (fd < 0) return -1; - while (nleft > 0) { - nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft); + nread = taosReadSocket(pSocket, ptr, (size_t)nleft); if (nread == 0) { break; } else if (nread < 0) { - if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) { + if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK*/) { continue; } else { return -1; @@ -324,11 +388,14 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { - taosSetNonblocking(fd, 1); +int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } + taosSetNonblocking(pSocket, 1); - int32_t nleft, nwritten, nready; - fd_set fset; + int32_t nleft, nwritten, nready; + fd_set fset; struct timeval tv; nleft = nbytes; @@ -336,24 +403,24 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { tv.tv_sec = 30; tv.tv_usec = 0; FD_ZERO(&fset); - FD_SET(fd, &fset); - if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + FD_SET(pSocket->fd, &fset); + if ((nready = select((SocketFd)(pSocket->fd + 1), NULL, &fset, NULL, &tv)) == 0) { errno = ETIMEDOUT; - //printf("fd %d timeout, no enough space to write", fd); + // printf("fd %d timeout, no enough space to write", fd); break; } else if (nready < 0) { if (errno == EINTR) continue; - //printf("select error, %d (%s)", errno, strerror(errno)); + // printf("select error, %d (%s)", errno, strerror(errno)); return -1; } - nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); + nwritten = (int32_t)send(pSocket->fd, ptr, (size_t)nleft, MSG_NOSIGNAL); if (nwritten <= 0) { if (errno == EAGAIN || errno == EINTR) continue; - //printf("write error, %d (%s)", errno, strerror(errno)); + // printf("write error, %d (%s)", errno, strerror(errno)); return -1; } @@ -361,121 +428,99 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { ptr += nwritten; } - taosSetNonblocking(fd, 0); + taosSetNonblocking(pSocket, 0); return (nbytes - nleft); } -int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { - int32_t nread, nready, nleft = nbytes; - - fd_set fset; - struct timeval tv; - - while (nleft > 0) { - tv.tv_sec = 30; - tv.tv_usec = 0; - FD_ZERO(&fset); - FD_SET(fd, &fset); - if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { - errno = ETIMEDOUT; - //printf("fd %d timeout\n", fd); - break; - } else if (nready < 0) { - if (errno == EINTR) continue; - //printf("select error, %d (%s)", errno, strerror(errno)); - return -1; - } - - if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { - if (errno == EINTR) continue; - //printf("read error, %d (%s)", errno, strerror(errno)); - return -1; - - } else if (nread == 0) { - //printf("fd %d EOF", fd); - break; // EOF - } - - nleft -= nread; - ptr += nread; - } - - return (nbytes - nleft); -} - -SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { +TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; - int32_t bufSize = 1024000; + SocketFd fd; + int32_t bufSize = 1024000; - //printf("open udp socket:0x%x:%hu", ip, port); + // printf("open udp socket:0x%x:%hu", ip, port); memset((char *)&localAddr, 0, sizeof(localAddr)); localAddr.sin_family = AF_INET; localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { - //printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; + if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + // printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(sockFd); - return -1; + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->fd = fd; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } /* bind socket to local address */ - if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { - //printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); - taosCloseSocket(sockFd); - return -1; + if (bind(pSocket->fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { + // printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); + taosCloseSocket(&pSocket); + return NULL; } - return sockFd; + return pSocket; } -SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; - int32_t ret; +TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + SocketFd fd = -1; + int32_t ret; struct sockaddr_in serverAddr, clientAddr; - int32_t bufSize = 1024 * 1024; + int32_t bufSize = 1024 * 1024; - sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - if (sockFd <= 2) { - //printf("failed to open the socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; + if (fd <= 2) { + // printf("failed to open the socket: %d (%s)", errno, strerror(errno)); + if (fd >= 0) taosCloseSocketNoCheck1(fd); + return NULL; } + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->fd = fd; + pSocket->refId = 0; + /* set REUSEADDR option, so the portnumber can be re-used */ int32_t reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - //printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the send buffer size for TCP socket\n"); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the send buffer size for TCP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the receive buffer size for TCP socket\n"); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the receive buffer size for TCP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } if (clientIp != 0) { @@ -485,11 +530,11 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie clientAddr.sin_port = 0; /* bind socket to client address */ - if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - //printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, - // strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (bind(pSocket->fd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { + // printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, + // strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } } @@ -498,159 +543,193 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie serverAddr.sin_addr.s_addr = destIp; serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); -#ifdef _TD_LINUX - taosSetNonblocking(sockFd, 1); - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#ifdef _TD_LINUX + taosSetNonblocking(pSocket, 1); + ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); if (ret == -1) { if (errno == EHOSTUNREACH) { - //printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(&pSocket); + return -1; } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { - struct pollfd wfd[1]; + struct pollfd wfd[1]; - wfd[0].fd = sockFd; + wfd[0].fd = pSocket->fd; wfd[0].events = POLLOUT; - + int res = poll(wfd, 1, TCP_CONN_TIMEOUT); if (res == -1 || res == 0) { - //printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); - taosCloseSocket(sockFd); // + // printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); + taosCloseSocket(&pSocket); // return -1; } - int optVal = -1, optLen = sizeof(int); - if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { - //printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); - taosCloseSocket(sockFd); // + int optVal = -1, optLen = sizeof(int); + if ((0 != taosGetSockOpt(pSocket, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { + // printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); + taosCloseSocket(&pSocket); // return -1; } ret = 0; - } else { // Other error - //printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); - taosCloseSocket(sockFd); // - return -1; - } + } else { // Other error + // printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); + taosCloseSocket(&pSocket); // + return -1; + } } - taosSetNonblocking(sockFd, 0); + taosSetNonblocking(pSocket, 0); #else - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); + ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); #endif if (ret != 0) { - //printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(sockFd); - sockFd = -1; + // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } else { - taosKeepTcpAlive(sockFd); + taosKeepTcpAlive(pSocket); } - return sockFd; + return pSocket; } -int32_t taosKeepTcpAlive(SOCKET sockFd) { +int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } int32_t alive = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { - //printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { + // printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } #ifndef __APPLE__ // all fails on macosx int32_t probes = 3; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { - //printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { + // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } int32_t alivetime = 10; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { - //printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { + // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } int32_t interval = 3; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { - //printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { + // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } -#endif // __APPLE__ +#endif // __APPLE__ int32_t nodelay = 1; - if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { - //printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { + // printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } struct linger linger = {0}; linger.l_onoff = 1; linger.l_linger = 3; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { - //printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { + // printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } return 0; } -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { +TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - SOCKET sockFd; + SocketFd fd; int32_t reuse; - //printf("open tcp server socket:0x%x:%hu", ip, port); + // printf("open tcp server socket:0x%x:%hu", ip, port); bzero((char *)&serverAdd, sizeof(serverAdd)); serverAdd.sin_family = AF_INET; serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { - //printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return NULL; } + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->refId = 0; + pSocket->fd = fd; + /* set REUSEADDR option, so the portnumber can be re-used */ reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - //printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } /* bind socket to server address */ - if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { - //printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - if (taosKeepTcpAlive(sockFd) < 0) { - //printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (taosKeepTcpAlive(pSocket) < 0) { + // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - if (listen(sockFd, 1024) < 0) { - //printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (listen(pSocket->fd, 1024) < 0) { + // printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - return sockFd; + return (TdSocketServerPtr)pSocket; } +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, + socklen_t *addrLen) { + if (pServerSocket == NULL || pServerSocket->fd < 0) { + return NULL; + } + SocketFd fd = accept(pServerSocket->fd, destAddr, addrLen); + if (fd == -1) { + // tError("TCP accept failure(%s)", strerror(errno)); + return NULL; + } + + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->fd = fd; + pSocket->refId = 0; + return pSocket; +} #define COPY_SIZE 32768 // sendfile shall be used -int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { +int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len) { + if (pSrcSocket == NULL || pSrcSocket->fd < 0 || pDestSocket == NULL || pDestSocket->fd < 0) { + return -1; + } int64_t leftLen; int64_t readLen, writeLen; char temp[COPY_SIZE]; @@ -663,18 +742,18 @@ int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { else readLen = COPY_SIZE; // 4K - int64_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); + int64_t retLen = taosReadMsg(pSrcSocket, temp, (int32_t)readLen); if (readLen != retLen) { - //printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, retLen, len, leftLen, strerror(errno)); + // printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + // readLen, retLen, len, leftLen, strerror(errno)); return -1; } - writeLen = taosWriteMsg(dfd, temp, (int32_t)readLen); + writeLen = taosWriteMsg(pDestSocket, temp, (int32_t)readLen); if (readLen != writeLen) { - //printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, writeLen, len, leftLen, strerror(errno)); + // printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + // readLen, writeLen, len, leftLen, strerror(errno)); return -1; } @@ -692,7 +771,7 @@ void taosBlockSIGPIPE() { sigaddset(&signal_mask, SIGPIPE); int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { - //printf("failed to block SIGPIPE"); + // printf("failed to block SIGPIPE"); } #endif } @@ -706,7 +785,7 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr * sa = result->ai_addr; + struct sockaddr *sa = result->ai_addr; struct sockaddr_in *si = (struct sockaddr_in *)sa; struct in_addr ia = si->sin_addr; uint32_t ip = ia.s_addr; @@ -715,12 +794,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { - //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); + // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); } else { - //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); } #else - //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); #endif return 0xFFFFFFFF; } @@ -730,7 +809,8 @@ int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; if (gethostname(hostname, 1023) == -1) { - //printf("failed to get hostname, reason:%s", strerror(errno)); + printf("failed to get hostname, reason:%s", strerror(errno)); + assert(0); return -1; } @@ -742,21 +822,22 @@ int32_t taosGetFqdn(char *fqdn) { // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return // immediately hints.ai_family = AF_INET; -#else // __APPLE__ +#else // __APPLE__ hints.ai_flags = AI_CANONNAME; -#endif // __APPLE__ +#endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - //printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + assert(0); return -1; } #ifdef __APPLE__ // refer to comments above strcpy(fqdn, hostname); -#else // __APPLE__ +#else // __APPLE__ strcpy(fqdn, result->ai_canonname); -#endif // __APPLE__ +#endif // __APPLE__ freeaddrinfo(result); return 0; } @@ -793,7 +874,6 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } - void taosIgnSIGPIPE() { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #else @@ -809,7 +889,67 @@ void taosSetMaskSIGPIPE() { sigaddset(&signal_mask, SIGPIPE); int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { - //printf("failed to setmask SIGPIPE"); + // printf("failed to setmask SIGPIPE"); } #endif -} \ No newline at end of file +} + +int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, socklen_t *addrLen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } + return getsockname(pSocket->fd, destAddr, addrLen); +} + + +TdEpollPtr taosCreateEpoll(int32_t size) { + EpollFd fd = -1; +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else + fd = epoll_create(size); +#endif + if (fd < 0) { + return NULL; + } + + TdEpollPtr pEpoll = (TdEpollPtr)malloc(sizeof(TdEpoll)); + if (pEpoll == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pEpoll->fd = fd; + pEpoll->refId = 0; + return pEpoll; +} +int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event) { + int32_t code = -1; + if (pEpoll == NULL || pEpoll->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else + code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event); +#endif + return code; +} +int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout) { + int32_t code = -1; + if (pEpoll == NULL || pEpoll->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else + code = epoll_wait(pEpoll->fd, event, maxEvents, timeout); +#endif + return code; +} +int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { + int32_t code; + if (ppEpoll == NULL || *ppEpoll == NULL || (*ppEpoll)->fd < 0) { + return -1; + } + code = taosCloseSocketNoCheck1((*ppEpoll)->fd); + (*ppEpoll)->fd = -1; + free(*ppEpoll); + return code; +} diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ff9d5fb71b..02d7e6c0e9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -16,6 +16,15 @@ #define _DEFAULT_SOURCE #include "os.h" +bool taosCheckSystemIsSmallEnd() { + union check{ + int16_t i; + char ch[2]; + }c; + c.i=1; + return c.ch[0]==1; +} + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index bb526e0ba0..6b60923189 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -79,7 +79,7 @@ static void* timer_routine(void* arg) { struct kevent64_s kev[10] = {0}; r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev) / sizeof(kev[0]), 0, &to); if (r != 0) { - fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); abort(); } timer_callback(SIGALRM); // just mock @@ -97,14 +97,14 @@ int taosInitTimer(void (*callback)(int), int ms) { timer_kq = kqueue(); if (timer_kq == -1) { - fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); } r = pthread_create(&timer_thread, NULL, timer_routine, NULL); if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); } @@ -116,7 +116,7 @@ void taosUninitTimer() { timer_stop = 1; r = pthread_join(timer_thread, NULL); if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); } diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 4b28800c28..7a47639e75 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -10,7 +10,7 @@ target_link_libraries( util PRIVATE os PUBLIC lz4_static - PUBLIC api cjson zlib + PUBLIC api cjson ) if(${BUILD_TEST}) diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index 105dd7f95f..d12b47efe8 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -32,7 +32,7 @@ void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set* #if 0 int64_t taosDesGenKey() { uint32_t iseed = (uint32_t)time(NULL); - srand(iseed); + taosSeedRand(iseed); uint8_t key[8] = {0}; generate_key(key); diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 6b89ed2c43..d9d6e4e3da 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -51,7 +51,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ pSkipList->len = keyLen; pSkipList->flags = flags; pSkipList->keyFn = fn; - pSkipList->seed = rand(); + pSkipList->seed = taosRand(); #if 0 // the function getkeycomparfunc is defined in common @@ -82,7 +82,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } } - srand((uint32_t)time(NULL)); + taosSeedRand((uint32_t)time(NULL)); #if SKIP_LIST_RECORD_PERFORMANCE pSkipList->state.nTotalMemSize += sizeof(SSkipList); @@ -560,9 +560,9 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { int32_t n = 1; #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) { + while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) { #else - while ((rand_r(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { + while ((taosRandR(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { #endif n++; } diff --git a/source/util/test/codingTests.cpp b/source/util/test/codingTests.cpp index 0cd9524646..b991411047 100644 --- a/source/util/test/codingTests.cpp +++ b/source/util/test/codingTests.cpp @@ -150,7 +150,7 @@ static bool test_variant_int64(int64_t value) { } TEST(codingTest, fixed_encode_decode) { - srand(time(0)); + taosSeedRand(time(0)); // uint16_t for (uint16_t value = 0; value <= UINT16_MAX; value++) { @@ -204,7 +204,7 @@ TEST(codingTest, fixed_encode_decode) { } TEST(codingTest, variant_encode_decode) { - srand(time(0)); + taosSeedRand(time(0)); // uint16_t for (uint16_t value = 0; value <= UINT16_MAX; value++) { diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index f392aac7d1..e63e6f04a1 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -161,7 +161,7 @@ void recyclePageTest() { TEST(testCase, resultBufferTest) { - srand(time(NULL)); + taosSeedRand(time(NULL)); simpleTest(); writeDownTest(); recyclePageTest(); diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp index f2e696b0e5..f61ebfd890 100644 --- a/source/util/test/skiplistTest.cpp +++ b/source/util/test/skiplistTest.cpp @@ -47,7 +47,7 @@ void doubleSkipListTest() { SSkipListKey sk; for (int32_t i = 0; i < 100; ++i) { sk.nType = TSDB_DATA_TYPE_DOUBLE; - int32_t idx = abs((i * rand()) % 1000); + int32_t idx = abs((i * taosRand()) % 1000); sk.dKey = doubleVal[idx]; @@ -74,7 +74,7 @@ void randKeyTest() { false, getkey); int32_t size = 200000; - srand(time(NULL)); + taosSeedRand(time(NULL)); printf("generated %d keys is: \n", size); @@ -87,7 +87,7 @@ void randKeyTest() { d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); - key[0] = rand() % 1000000000; + key[0] = taosRand() % 1000000000; key[1] = key[0]; @@ -337,7 +337,7 @@ void duplicatedKeyTest() { TEST(testCase, skiplist_test) { assert(sizeof(SSkipListKey) == 8); - srand(time(NULL)); + taosSeedRand(time(NULL)); stringKeySkiplistTest(); doubleSkipListTest(); diff --git a/tests/test-all.sh b/tests/test-all.sh index c2bc305de0..aa7c4240bc 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -1,5 +1,7 @@ #!/bin/bash +# set -x + # Color setting RED='\033[0;31m' GREEN='\033[1;32m' @@ -12,19 +14,19 @@ tests_dir=`pwd` IN_TDINTERNAL="community" function stopTaosd { - echo "Stop taosd" + echo "Stop taosd" sudo systemctl stop taosd || echo 'no sudo or systemctl or stop fail' PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do + while [ -n "$PID" ] + do pkill -TERM -x taosd sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - done + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done } function dohavecore(){ - corefile=`find $corepath -mmin 1` + [ -d $corepath ] && corefile=`find $corepath -mmin 1` || return 1 if [ -n "$corefile" ];then core_file=`echo $corefile|cut -d " " -f2` proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` @@ -39,9 +41,9 @@ function dohavecore(){ cd community cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi - else + else cd ../../ - if [[ $1 == 1 ]];then + if [[ $1 == 1 ]];then #tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi @@ -56,20 +58,20 @@ function dohavecore(){ function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` - start_time=`date +%s` + case=`echo $line | grep sim$ |awk '{print $NF}'` + start_time=`date +%s` date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then echo -n $case - ./test.sh -f $case > /dev/null 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ./test.sh -f $case > /dev/null 2>&1 \ + && ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log ) \ + || ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ echo -e "${RED} failed${NC}" | tee -a out.log else echo -n $case ./test.sh -f $case > /dev/null 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ echo -e "${RED} failed${NC}" | tee -a out.log fi out_log=`tail -1 out.log ` @@ -85,35 +87,35 @@ function runSimCaseOneByOne { function runSimCaseOneByOnefq { start=`sed -n "/$1-start/=" jenkins/basic.txt` - end=`sed -n "/$1-end/=" jenkins/basic.txt` + end=`sed -n "/$1-end/=" jenkins/basic.txt` for ((i=$start;i<=$end;i++)) ; do line=`sed -n "$i"p jenkins/basic.txt` if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` + case=`echo $line | grep sim$ |awk '{print $NF}'` - start_time=`date +%s` + start_time=`date +%s` date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then echo -n $case - ./test.sh -f $case > case.log 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ./test.sh -f $case > case.log 2>&1 \ + && \ + ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else - pwd echo -n $case ./test.sh -f $case > ../../sim/case.log 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && pwd && cat ../../sim/case.log ) fi - + out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` - else + else cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi dohavecore $2 1 @@ -125,7 +127,7 @@ function runSimCaseOneByOnefq { echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log dohavecore $2 1 fi - done + done rm -rf ../../../sim/case.log rm -rf ../../sim/case.log } @@ -134,7 +136,6 @@ function runPyCaseOneByOne { while read -r line; do if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then - if [[ $line =~ '-r' ]];then case=`echo $line|awk '{print $4}'` else @@ -172,7 +173,6 @@ function runPyCaseOneByOnefq() { line=`sed -n "$i"p fulltest.sh` if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then - if [[ $line =~ '-r' ]];then case=`echo $line|awk '{print $4}'` else @@ -186,7 +186,7 @@ function runPyCaseOneByOnefq() { fi $line > case.log 2>&1 && \ echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ - echo -e "${RED} failed${NC}" | tee -a pytest-out.log + echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then @@ -205,7 +205,7 @@ function runPyCaseOneByOnefq() { fi dohavecore $2 2 fi - done + done rm -rf ../../sim/case.log } @@ -239,7 +239,9 @@ totalExampleFailed=0 if [ "${OS}" == "Linux" ]; then corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` if [ -z "$corepath" ];then - echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied" + [ -d /coredump ] || mkdir /coredump || echo -e "failed to mdkir /coredump" + [ -d /coredump ] \ + && echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied" corepath="/coredump/" fi fi @@ -248,6 +250,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" ! echo "### run TSIM test case ###" cd $tests_dir/script + [ -d ../../sim ] || mkdir ../../sim || echo -e "failed to mkdir ../../sim" [ -f out.log ] && rm -f out.log if [ "$1" == "cron" ]; then echo "### run TSIM regression test ###" @@ -358,7 +361,7 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != " runPyCaseOneByOnefq p2 1 elif [ "$1" == "p3" ]; then echo "### run Python_3 test ###" - runPyCaseOneByOnefq p3 1 + runPyCaseOneByOnefq p3 1 elif [ "$1" == "p4" ]; then echo "### run Python_4 test ###" runPyCaseOneByOnefq p4 1 @@ -368,16 +371,20 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != " echo "### run Python smoke test ###" runPyCaseOneByOne smoketest.sh fi - totalPySuccess=`grep 'success' pytest-out.log | wc -l` - if [ "$totalPySuccess" -gt "0" ]; then - echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" - fi + if [ -f pytest-out.log ]; then + totalPySuccess=`grep 'success' pytest-out.log | wc -l` + if [ "$totalPySuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" + fi - totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` - if [ "$totalPyFailed" -ne "0" ]; then - echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" -# exit $totalPyFailed + totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` + if [ "$totalPyFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" + # exit $totalPyFailed + fi + else + echo -e "pytest-out.log not found!" fi fi @@ -395,14 +402,14 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != pwd cd debug/ - + stopTaosd rm -rf /var/lib/taos/* nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 & sleep 30 - - cd $tests_dir/../src/connector/jdbc - + + cd $tests_dir/../src/connector/jdbc + mvn test > jdbc-out.log 2>&1 tail -n 20 jdbc-out.log @@ -412,14 +419,14 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != JDBCFailed=`echo ${failed/%,}` error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'` JDBCError=`echo ${error/%,}` - + totalJDBCFailed=`expr $JDBCFailed + $JDBCError` totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed` if [ "$totalJDBCSuccess" -gt "0" ]; then echo -e "\n${GREEN} ### Total $totalJDBCSuccess JDBC case(s) succeed! ### ${NC}" fi - + if [ "$totalJDBCFailed" -ne "0" ]; then echo -e "\n${RED} ### Total $totalJDBCFailed JDBC case(s) failed! ### ${NC}" fi @@ -427,7 +434,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != fi if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "example" ] && [ "$1" == "full" ]; then - echo "### run Unit tests ###" + echo "### run Unit tests ###" stopTaosd cd $tests_dir @@ -443,19 +450,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != rm -rf /var/lib/taos/* nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & sleep 30 - + pwd ./queryTest > unittest-out.log 2>&1 tail -n 20 unittest-out.log - totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` + totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'` totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess` if [ "$totalUnitSuccess" -gt "0" ]; then echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}" fi - + if [ "$totalUnitFailed" -ne "0" ]; then echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}" fi @@ -463,7 +470,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != fi if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then - echo "### run Example tests ###" + echo "### run Example tests ###" stopTaosd cd $tests_dir @@ -480,7 +487,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & echo "sleeping for 30 seconds" #sleep 30 - + cd $tests_dir echo "current dir: " pwd @@ -493,16 +500,16 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./apitest > /dev/null 2>&1 if [ $? != "0" ]; then echo "apitest failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "apitest pass" totalExamplePass=`expr $totalExamplePass + 1` - fi + fi ./prepare 127.0.0.1 > /dev/null 2>&1 if [ $? != "0" ]; then echo "prepare failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "prepare pass" totalExamplePass=`expr $totalExamplePass + 1` @@ -511,7 +518,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./subscribe -test > /dev/null 2>&1 if [ $? != "0" ]; then echo "subscribe failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "subscribe pass" totalExamplePass=`expr $totalExamplePass + 1` @@ -520,7 +527,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1 if [ $? != "0" ]; then echo "asyncdemo failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "asyncdemo pass" totalExamplePass=`expr $totalExamplePass + 1` @@ -529,16 +536,16 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./demo 127.0.0.1 > /dev/null 2>&1 if [ $? != "0" ]; then echo "demo failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "demo pass" totalExamplePass=`expr $totalExamplePass + 1` fi - + if [ "$totalExamplePass" -gt "0" ]; then echo -e "\n${GREEN} ### Total $totalExamplePass examples succeed! ### ${NC}" fi - + if [ "$totalExampleFailed" -ne "0" ]; then echo -e "\n${RED} ### Total $totalExampleFailed examples failed! ### ${NC}" fi diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 3fe0425c91..cb7d7c67ce 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -192,11 +192,11 @@ static void msg_process(tmq_message_t* message) { tmqShowMsg(message); } // calc dir size (not include itself 4096Byte) int64_t getDirectorySize(char *dir) { - DIR *dp; - struct dirent *entry; + TdDirPtr pDir; + TdDirEntryPtr pDirEntry; int64_t totalSize=0; - if ((dp = opendir(dir)) == NULL) { + if ((pDir = taosOpenDir(dir)) == NULL) { fprintf(stderr, "Cannot open dir: %s\n", dir); return -1; } @@ -204,26 +204,27 @@ int64_t getDirectorySize(char *dir) //lstat(dir, &statbuf); //totalSize+=statbuf.st_size; - while ((entry = readdir(dp)) != NULL) { + while ((pDirEntry = taosReadDir(pDir)) != NULL) { char subdir[1024]; - sprintf(subdir, "%s/%s", dir, entry->d_name); + char* fileName = taosGetDirEntryName(pDirEntry); + sprintf(subdir, "%s/%s", dir, fileName); //printf("===d_name: %s\n", entry->d_name); if (taosIsDir(subdir)) { - if (strcmp(".", entry->d_name) == 0 || strcmp("..", entry->d_name) == 0) { + if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) { continue; } int64_t subDirSize = getDirectorySize(subdir); totalSize+=subDirSize; - } else if (0 == strcmp(strchr(entry->d_name, '.'), ".log")) { // only calc .log file size, and not include .idx file + } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file int64_t file_size = 0; taosStatFile(subdir, &file_size, NULL); totalSize+=file_size; } } - closedir(dp); + taosCloseDir(pDir); return totalSize; } @@ -352,7 +353,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { int32_t cnt = 0; /*clock_t startTime = clock();*/ while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { cnt++; msg_process(tmqmessage); @@ -383,7 +384,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { msg_process(tmqmessage); tmq_message_destroy(tmqmessage); @@ -411,7 +412,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog int32_t skipLogNum = 0; int64_t startTime = taosGetTimestampUs(); while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { batchCnt++; skipLogNum += tmqGetSkipLogNum(tmqmessage);