diff --git a/cmake/cmake.options b/cmake/cmake.options index e84d02800c..e19c10f6b2 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -47,13 +47,13 @@ option( option( BUILD_WITH_UV "If build with libuv" - OFF + ON ) option( BUILD_WITH_UV_TRANS "If build with libuv_trans " - OFF + ON ) option( diff --git a/example/src/tmq.c b/example/src/tmq.c index 26e5ea82c1..094fd94bfc 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -66,7 +66,7 @@ int32_t init_env() { } int32_t create_topic() { - printf("create topic"); + printf("create topic\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -91,6 +91,10 @@ int32_t create_topic() { return 0; } +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d\n", resp); +} + tmq_t* build_consumer() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -103,6 +107,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); return tmq; } @@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1000; + static const int MIN_COMMIT_COUNT = 1; int msg_count = 0; tmq_resp_err_t err; @@ -214,6 +219,6 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); } diff --git a/include/client/taos.h b/include/client/taos.h index a960d37937..2c8135c8ff 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi /* --------------------------TMQ INTERFACE------------------------------- */ enum tmq_resp_err_t { + TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__SUCCESS = 0, - TMQ_RESP_ERR__FAIL = 1, }; typedef enum tmq_resp_err_t tmq_resp_err_t; @@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); #if 0 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); @@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t * #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); #endif +DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { diff --git a/include/common/common.h b/include/common/common.h index fd5b6717ab..8fa2d03d6d 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,7 +16,6 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H - #ifdef __cplusplus extern "C" { #endif @@ -43,14 +42,16 @@ extern "C" { // int16_t bytes; // } SSchema; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 -#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 +enum { + TMQ_CONF__RESET_OFFSET__LATEST = -1, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__NONE = -3, +}; typedef struct { uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + SArray* pGroupList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; typedef struct SColumnDataAgg { @@ -79,14 +80,14 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray + SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { - int32_t *offset; // start position for each entry in the list + int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data uint32_t allocLen; // allocated buffer size } SVarColAttr; @@ -94,11 +95,11 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - bool hasNull;// if current column data has null value. - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + bool hasNull; // if current column data has null value. + char* pData; // the corresponding block data in memory union { - char *nullbitmap; // bitmap, one bit for each item in the list + char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; } SColumnInfoData; @@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); @@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); @@ -250,11 +249,11 @@ typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - int32_t numOfCols; - SColumn* pColumns; // data columns that are required by query - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SExprInfo { @@ -271,7 +270,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/tcompare.h b/include/common/tcompare.h index 8476a79e92..5dcbf37b90 100644 --- a/include/common/tcompare.h +++ b/include/common/tcompare.h @@ -23,8 +23,12 @@ extern "C" { #endif -int32_t compareStrPatternComp(const void* pLeft, const void* pRight); -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight); +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight); + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight); +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight); + __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); @@ -33,4 +37,4 @@ int32_t doCompare(const char* a, const char* b, int32_t type, size_t size) } #endif -#endif /*_TD_TCOMPARE_H_*/ \ No newline at end of file +#endif /*_TD_TCOMPARE_H_*/ diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5164fb6ccd..ae3586e735 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -422,8 +422,8 @@ typedef struct { } SColumnInfo; typedef struct { - uint64_t uid; - TSKEY key; // last accessed ts, for subscription + int64_t uid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -554,8 +554,8 @@ int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; } SDropDbRsp; int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); @@ -570,12 +570,12 @@ int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SArray* pVgroupInfos; // Array of SVgroupInfo + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); @@ -725,7 +725,7 @@ typedef struct { int32_t vgId; int32_t dnodeId; char db[TSDB_DB_FNAME_LEN]; - uint64_t dbUid; + int64_t dbUid; int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; @@ -753,10 +753,10 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); typedef struct { - int32_t vgId; - int32_t dnodeId; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; + int32_t vgId; + int32_t dnodeId; + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); @@ -796,7 +796,7 @@ typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - uint64_t dbId; + int64_t dbId; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -1268,7 +1268,7 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; int32_t sverson; int32_t execLen; char* executor; @@ -1279,11 +1279,11 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; } SDDropTopicReq; typedef struct SVCreateTbReq { - uint64_t ver; // use a general definition + int64_t ver; // use a general definition char* name; uint32_t ttl; uint32_t keep; @@ -1314,8 +1314,8 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); typedef struct { - uint64_t ver; // use a general definition - SArray* pArray; + int64_t ver; // use a general definition + SArray* pArray; } SVCreateTbBatchReq; typedef struct { @@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - uint64_t ver; + int64_t ver; char* name; uint8_t type; tb_uid_t suid; @@ -1760,35 +1760,19 @@ typedef struct { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqOffset; -typedef struct { - int32_t vgId; - SArray* offsets; // SArray -} SMqVgOffsets; - typedef struct { int32_t num; SMqOffset* offsets; -} SMqCMResetOffsetReq; +} SMqCMCommitOffsetReq; typedef struct { int32_t reserved; -} SMqCMResetOffsetRsp; - -typedef struct { - int64_t leftForVer; - SMqVgOffsets offsets; -} SMqMVResetOffsetReq; - -typedef struct { - int32_t reserved; -} SMqMVResetOffsetRsp; +} SMqCMCommitOffsetRsp; int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset); int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset); -int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq); -int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq); -int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq); -int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq); +int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); +int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); typedef struct { uint32_t nCols; @@ -1870,7 +1854,6 @@ typedef struct { typedef struct { int64_t consumerId; SSchemaWrapper* schemas; - int64_t committedOffset; int64_t reqOffset; int64_t rspOffset; int32_t skipLogNum; @@ -1881,22 +1864,18 @@ typedef struct { // one req for one vg+topic typedef struct { SMsgHead head; - // 0: commit only, current offset - // 1: consume only, poll next offset - // 2: commit current and consume next offset - int32_t reqType; - int64_t reqId; int64_t consumerId; int64_t blockingTime; char cgroup[TSDB_CONSUMER_GROUP_LEN]; - int64_t offset; + int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; typedef struct { int32_t vgId; + int64_t offset; SEpSet epSet; } SMqSubVgEp; @@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return buf; } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index f43641c6f5..a3e44968f7 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -144,10 +144,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -195,4 +195,3 @@ enum { TDMT_MAX #endif }; -// clang-format on diff --git a/include/common/tname.h b/include/common/tname.h index 12a0d34cb4..47028fbce1 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,6 +16,11 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H +#ifdef __cplusplus +extern "C" { +#endif + + #include "tdef.h" #include "tmsg.h" @@ -59,4 +64,9 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId); SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +#ifdef __cplusplus +} +#endif + + #endif // TDENGINE_TNAME_H diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 505685f8d5..39a5f0e2b8 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -78,9 +78,12 @@ typedef struct { case TSDB_DATA_TYPE_UINT: \ (_v) = (_finalType)GET_UINT32_VAL(_data); \ break; \ - default: \ + case TSDB_DATA_TYPE_INT: \ (_v) = (_finalType)GET_INT32_VAL(_data); \ break; \ + default: \ + (_v) = (_finalType)varDataLen(_data); \ + break; \ } \ } while (0) @@ -88,6 +91,8 @@ typedef struct { do { \ switch (_type) { \ case TSDB_DATA_TYPE_BOOL: \ + *(bool *)(_v) = (bool)(_data); \ + break; \ case TSDB_DATA_TYPE_TINYINT: \ *(int8_t *)(_v) = (int8_t)(_data); \ break; \ @@ -101,6 +106,7 @@ typedef struct { *(uint16_t *)(_v) = (uint16_t)(_data); \ break; \ case TSDB_DATA_TYPE_BIGINT: \ + case TSDB_DATA_TYPE_TIMESTAMP: \ *(int64_t *)(_v) = (int64_t)(_data); \ break; \ case TSDB_DATA_TYPE_UBIGINT: \ @@ -115,9 +121,11 @@ typedef struct { case TSDB_DATA_TYPE_UINT: \ *(uint32_t *)(_v) = (uint32_t)(_data); \ break; \ - default: \ + case TSDB_DATA_TYPE_INT: \ *(int32_t *)(_v) = (int32_t)(_data); \ break; \ + default: \ + break; \ } \ } while (0) @@ -138,6 +146,9 @@ typedef struct { #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) +#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) +#define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) + static FORCE_INLINE bool isNull(const void *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -205,6 +216,7 @@ void* getDataMax(int32_t type); #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) +#define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL) #ifdef __cplusplus } diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 1e967a6d2b..bf48a8523c 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,14 +113,15 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_SUBSCRIBE = 10, - SDB_CONSUMER = 11, - SDB_TOPIC = 12, - SDB_VGROUP = 13, - SDB_STB = 14, - SDB_DB = 15, - SDB_FUNC = 16, - SDB_MAX = 17 + SDB_OFFSET = 10, + SDB_SUBSCRIBE = 11, + SDB_CONSUMER = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index aef5f7fec4..e970a0d693 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -226,13 +226,19 @@ typedef struct SAggFunctionInfo { int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; -struct SScalarFuncParam; +typedef struct SScalarParam { + void* data; + bool colData; + int32_t num; + int32_t type; + int32_t bytes; +} SScalarParam; typedef struct SScalarFunctionInfo { char name[FUNCTIONS_NAME_MAX_LENGTH]; int8_t type; // scalar function or aggregation function uint32_t functionId; // index of scalar function - void (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput); + void (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput); } SScalarFunctionInfo; typedef struct SMultiFunctionsDesc { @@ -285,10 +291,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); -struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num); -void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num); -struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index); - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // fill api struct SFillInfo; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 69673804fe..e77ccaa225 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "querynodes.h" +#include "function.h" typedef enum EFunctionType { // aggregate function @@ -117,6 +118,13 @@ typedef struct SFuncExecFuncs { FExecFinalize finalize; } SFuncExecFuncs; +typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +typedef struct SScalarFuncExecFuncs { + FScalarExecProcess process; +} SScalarFuncExecFuncs; + + int32_t fmFuncMgtInit(); int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); @@ -134,6 +142,7 @@ bool fmIsTimeorderFunc(int32_t funcId); int32_t fmFuncScanType(int32_t funcId); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); +int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); #ifdef __cplusplus } diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6a0422cd80..f09ffa8926 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -61,22 +61,27 @@ typedef enum ENodeType { QUERY_NODE_INTERVAL_WINDOW, QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, + QUERY_NODE_RAW_EXPR, // Only be used in parser module. QUERY_NODE_COLUMN_REF, QUERY_NODE_TARGET, - - // Only be used in parser module. - QUERY_NODE_RAW_EXPR, + QUERY_NODE_TUPLE_DESC, + QUERY_NODE_SLOT_DESC, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, QUERY_NODE_SHOW_STMT, + // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN, QUERY_NODE_LOGIC_PLAN_JOIN, - QUERY_NODE_LOGIC_PLAN_FILTER, QUERY_NODE_LOGIC_PLAN_AGG, - QUERY_NODE_LOGIC_PLAN_PROJECT + QUERY_NODE_LOGIC_PLAN_PROJECT, + + // physical plan node + QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, + QUERY_NODE_PHYSICAL_PLAN_PROJECT } ENodeType; /** @@ -132,7 +137,7 @@ bool nodesEqualNode(const SNode* a, const SNode* b); SNode* nodesCloneNode(const SNode* pNode); SNodeList* nodesCloneList(const SNodeList* pList); -int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); +int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); #ifdef __cplusplus diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3668f256eb..40684f2b26 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "querynodes.h" +#include "tmsg.h" typedef struct SLogicNode { ENodeType type; @@ -31,10 +32,20 @@ typedef struct SLogicNode { struct SLogicNode* pParent; } SLogicNode; +typedef enum EScanType { + SCAN_TYPE_TAG, + SCAN_TYPE_TABLE, + SCAN_TYPE_STABLE, + SCAN_TYPE_STREAM +} EScanType; + typedef struct SScanLogicNode { SLogicNode node; SNodeList* pScanCols; struct STableMeta* pMeta; + EScanType scanType; + uint8_t scanFlag; // denotes reversed scan of data or not + STimeWindow scanRange; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -43,10 +54,6 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; } SJoinLogicNode; -typedef struct SFilterLogicNode { - SLogicNode node; -} SFilterLogicNode; - typedef struct SAggLogicNode { SLogicNode node; SNodeList* pGroupKeys; @@ -58,6 +65,56 @@ typedef struct SProjectLogicNode { SNodeList* pProjections; } SProjectLogicNode; +typedef struct SSlotDescNode { + ENodeType type; + int16_t slotId; + SDataType dataType; + int16_t srcTupleId; + int16_t srcSlotId; + bool reserve; + bool output; +} SSlotDescNode; + +typedef struct STupleDescNode { + ENodeType type; + int16_t tupleId; + SNodeList* pSlots; +} STupleDescNode; + +typedef struct SPhysiNode { + ENodeType type; + STupleDescNode outputTuple; + SNode* pConditions; + SNodeList* pChildren; + struct SPhysiNode* pParent; +} SPhysiNode; + +typedef struct SScanPhysiNode { + SPhysiNode node; + SNodeList* pScanCols; + uint64_t uid; // unique id of the table + int8_t tableType; + int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC + int32_t count; // repeat count + int32_t reverse; // reverse scan count +} SScanPhysiNode; + +typedef SScanPhysiNode SSystemTableScanPhysiNode; +typedef SScanPhysiNode STagScanPhysiNode; + +typedef struct STableScanPhysiNode { + SScanPhysiNode scan; + uint8_t scanFlag; // denotes reversed scan of data or not + STimeWindow scanRange; +} STableScanPhysiNode; + +typedef STableScanPhysiNode STableSeqScanPhysiNode; + +typedef struct SProjectPhysiNode { + SPhysiNode node; + SNodeList* pProjections; +} SProjectPhysiNode; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 024967289a..7bc8617db1 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -68,6 +68,13 @@ typedef struct SColumnRefNode { int16_t columnId; } SColumnRefNode; +typedef struct STargetNode { + ENodeType type; + int16_t tupleId; + int16_t slotId; + SNode* pExpr; +} STargetNode; + typedef struct SValueNode { SExprNode node; // QUERY_NODE_VALUE char* literal; @@ -81,45 +88,6 @@ typedef struct SValueNode { } datum; } SValueNode; -typedef enum EOperatorType { - // arithmetic operator - OP_TYPE_ADD = 1, - OP_TYPE_SUB, - OP_TYPE_MULTI, - OP_TYPE_DIV, - OP_TYPE_MOD, - - // bit operator - OP_TYPE_BIT_AND, - OP_TYPE_BIT_OR, - - // comparison operator - OP_TYPE_GREATER_THAN, - OP_TYPE_GREATER_EQUAL, - OP_TYPE_LOWER_THAN, - OP_TYPE_LOWER_EQUAL, - OP_TYPE_EQUAL, - OP_TYPE_NOT_EQUAL, - OP_TYPE_IN, - OP_TYPE_NOT_IN, - OP_TYPE_LIKE, - OP_TYPE_NOT_LIKE, - OP_TYPE_MATCH, - OP_TYPE_NMATCH, - OP_TYPE_IS_NULL, - OP_TYPE_IS_NOT_NULL, - OP_TYPE_IS_TRUE, - OP_TYPE_IS_FALSE, - OP_TYPE_IS_UNKNOWN, - OP_TYPE_IS_NOT_TRUE, - OP_TYPE_IS_NOT_FALSE, - OP_TYPE_IS_NOT_UNKNOWN, - - // json operator - OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS -} EOperatorType; - typedef struct SOperatorNode { SExprNode node; // QUERY_NODE_OPERATOR EOperatorType opType; @@ -127,11 +95,6 @@ typedef struct SOperatorNode { SNode* pRight; } SOperatorNode; -typedef enum ELogicConditionType { - LOGIC_COND_TYPE_AND, - LOGIC_COND_TYPE_OR, - LOGIC_COND_TYPE_NOT, -} ELogicConditionType; typedef struct SLogicConditionNode { SExprNode node; // QUERY_NODE_LOGIC_CONDITION @@ -141,6 +104,7 @@ typedef struct SLogicConditionNode { typedef struct SNodeListNode { ENodeType type; // QUERY_NODE_NODE_LIST + SDataType dataType; SNodeList* pNodeList; } SNodeListNode; @@ -307,8 +271,10 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); +void* nodesGetValueFromNode(SValueNode *pNode); + #ifdef __cplusplus } #endif -#endif /*_TD_QUERY_NODES_H_*/ \ No newline at end of file +#endif /*_TD_QUERY_NODES_H_*/ diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h new file mode 100644 index 0000000000..fd57016ec3 --- /dev/null +++ b/include/libs/scalar/filter.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_FILTER_H +#define TDENGINE_FILTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SFilterInfo SFilterInfo; +typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); + + +enum { + FLT_OPTION_NO_REWRITE = 1, + FLT_OPTION_TIMESTAMP = 2, + FLT_OPTION_NEED_UNIQE = 4, +}; + +typedef struct SFilterColumnParam{ + int32_t numOfCols; + SArray* pDataBlock; +} SFilterColumnParam; + + +extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); +extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); +extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp); +extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp); +extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); +extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); +extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); +extern void filterFreeInfo(SFilterInfo *info); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_FILTER_H diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h new file mode 100644 index 0000000000..1902c11cc6 --- /dev/null +++ b/include/libs/scalar/scalar.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_SCALAR_H +#define TDENGINE_SCALAR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "function.h" +#include "nodes.h" +#include "querynodes.h" + +typedef struct SFilterInfo SFilterInfo; + + +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); +int32_t scalarGetOperatorParamNum(EOperatorType type); +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); + +int32_t vectorGetConvertType(int32_t type1, int32_t type2); +int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SCALAR_H diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 00ba1120e7..0ec741ec3e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -137,22 +137,21 @@ typedef struct { } SSyncInfo; -// will be defined in syncInt.h, here just for complie -typedef struct SSyncNode { -} SSyncNode; +struct SSyncNode; +typedef struct SSyncNode SSyncNode; int32_t syncInit(); void syncCleanUp(); -int64_t syncStart(const SSyncInfo*); +int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg*); +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); // int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); ESyncState syncGetMyRole(int64_t rid); -void syncGetNodesRole(int64_t rid, SNodesRole*); +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); extern int32_t sDebugFlag; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index f913ba06d0..538aeb1a0e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -64,6 +64,7 @@ typedef struct SRpcInit { int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int idleTime; // milliseconds, 0 means idle timer is disabled + bool noPool; // create conn pool or not // the following is for client app ecurity only char *user; // user name char spi; // security parameter index diff --git a/include/util/compare.h b/include/util/compare.h index 70a8134b35..6305334e02 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -49,10 +49,19 @@ int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, int32_t taosArrayCompareString(const void *a, const void *b); -int32_t setCompareBytes1(const void *pLeft, const void *pRight); -int32_t setCompareBytes2(const void *pLeft, const void *pRight); -int32_t setCompareBytes4(const void *pLeft, const void *pRight); -int32_t setCompareBytes8(const void *pLeft, const void *pRight); +int32_t setChkInBytes1(const void *pLeft, const void *pRight); +int32_t setChkInBytes2(const void *pLeft, const void *pRight); +int32_t setChkInBytes4(const void *pLeft, const void *pRight); +int32_t setChkInBytes8(const void *pLeft, const void *pRight); + +int32_t setChkNotInBytes1(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes2(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes4(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes8(const void *pLeft, const void *pRight); + +int32_t compareChkInString(const void *pLeft, const void *pRight); +int32_t compareChkNotInString(const void *pLeft, const void *pRight); + int32_t compareInt8Val(const void *pLeft, const void *pRight); int32_t compareInt16Val(const void *pLeft, const void *pRight); @@ -74,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight); int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); -int32_t compareFindItemInSet(const void *pLeft, const void *pRight); int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); @@ -92,6 +100,9 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); +__compar_fn_t getComparFunc(int32_t type, int32_t optr); + + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3f6db72c80..2f7d406569 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -20,6 +20,8 @@ extern "C" { #endif +// clang-format off + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) @@ -262,6 +264,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode diff --git a/include/util/tdef.h b/include/util/tdef.h index f57d9b6f35..d0f2b77f1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H @@ -108,33 +110,52 @@ do { \ (src) = (void *)((char *)src + sizeof(type));\ } while(0) +typedef enum EOperatorType { + // arithmetic operator + OP_TYPE_ADD = 1, + OP_TYPE_SUB, + OP_TYPE_MULTI, + OP_TYPE_DIV, + OP_TYPE_MOD, -// TODO: check if below is necessary -#define TSDB_RELATION_INVALID 0 -#define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_GREATER 2 -#define TSDB_RELATION_EQUAL 3 -#define TSDB_RELATION_LESS_EQUAL 4 -#define TSDB_RELATION_GREATER_EQUAL 5 -#define TSDB_RELATION_NOT_EQUAL 6 -#define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_ISNULL 8 -#define TSDB_RELATION_NOTNULL 9 -#define TSDB_RELATION_IN 10 + // bit operator + OP_TYPE_BIT_AND, + OP_TYPE_BIT_OR, -#define TSDB_RELATION_AND 11 -#define TSDB_RELATION_OR 12 -#define TSDB_RELATION_NOT 13 + // comparison operator + OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_EQUAL, + OP_TYPE_LOWER_THAN, + OP_TYPE_LOWER_EQUAL, + OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, + OP_TYPE_IN, + OP_TYPE_NOT_IN, + OP_TYPE_LIKE, + OP_TYPE_NOT_LIKE, + OP_TYPE_MATCH, + OP_TYPE_NMATCH, + OP_TYPE_IS_NULL, + OP_TYPE_IS_NOT_NULL, + OP_TYPE_IS_TRUE, + OP_TYPE_IS_FALSE, + OP_TYPE_IS_UNKNOWN, + OP_TYPE_IS_NOT_TRUE, + OP_TYPE_IS_NOT_FALSE, + OP_TYPE_IS_NOT_UNKNOWN, -#define TSDB_RELATION_MATCH 14 -#define TSDB_RELATION_NMATCH 15 + // json operator + OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_CONTAINS +} EOperatorType; + + +typedef enum ELogicConditionType { + LOGIC_COND_TYPE_AND, + LOGIC_COND_TYPE_OR, + LOGIC_COND_TYPE_NOT, +} ELogicConditionType; -#define TSDB_BINARY_OP_ADD 4000 -#define TSDB_BINARY_OP_SUBTRACT 4001 -#define TSDB_BINARY_OP_MULTIPLY 4002 -#define TSDB_BINARY_OP_DIVIDE 4003 -#define TSDB_BINARY_OP_REMAINDER 4004 -#define TSDB_BINARY_OP_CONCAT 4005 #define FUNCTION_CEIL 4500 #define FUNCTION_FLOOR 4501 @@ -146,9 +167,6 @@ do { \ #define FUNCTION_LTRIM 4802 #define FUNCTION_RTRIM 4803 -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) -#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) - #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -180,6 +198,7 @@ do { \ #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE @@ -333,9 +352,6 @@ do { \ #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode - #define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default diff --git a/include/util/thash.h b/include/util/thash.h index 3a614a73a6..1f72045a83 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -55,6 +55,8 @@ uint32_t taosIntHash_64(const char *key, uint32_t len); _hash_fn_t taosGetDefaultHashFunction(int32_t type); +_equal_fn_t taosGetDefaultEqualFunction(int32_t type); + typedef struct SHashNode { struct SHashNode *next; uint32_t hashVal; // the hash value of key @@ -258,6 +260,8 @@ void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashRelease(SHashObj *pHashObj, void *p); +void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); + #ifdef __cplusplus } diff --git a/include/util/tjson.h b/include/util/tjson.h index a4eb6e5385..a0c2fef05b 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -30,6 +30,7 @@ void tjsonDelete(SJson* pJson); SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number); +int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number); int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal); int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); @@ -42,6 +43,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj); typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj); char* tjsonToString(const SJson* pJson); +char* tjsonToUnformattedString(const SJson* pJson); #ifdef __cplusplus } diff --git a/include/util/types.h b/include/util/types.h index cb25448cc4..25688df56c 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -66,6 +66,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { // #else #define GET_FLOAT_VAL(x) (*(float *)(x)) #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_BIGINT_VAL(x, y) { (*(int64_t *)(x)) = (int64_t)(y); } #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 229d3a9ec3..9a1025c4bd 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -45,22 +45,24 @@ struct tmq_topic_vgroup_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - bool auto_commit; + char clientId[256]; + char groupId[256]; + int8_t auto_commit; + int8_t resetOffset; + tmq_commit_cb* commit_cb; /*char* ip;*/ /*uint16_t port;*/ - tmq_commit_cb* commit_cb; }; struct tmq_t { // conf char groupId[256]; char clientId[256]; - bool autoCommit; + int8_t autoCommit; SRWLatch lock; int64_t consumerId; int32_t epoch; + int32_t resetOffsetCfg; int64_t status; tsem_t rspSem; STscObj* pTscObj; @@ -79,7 +81,6 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -115,21 +116,17 @@ typedef struct { } SMqConsumeCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; -} SMqCommitCbParam; - -typedef struct { - tmq_t* tmq; + tmq_t* tmq; + /*SMqClientVg* pVg;*/ + int32_t async; tsem_t rspSem; tmq_resp_err_t rspErr; -} SMqResetOffsetParam; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; return conf; } @@ -157,6 +154,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { + if (strcmp(value, "none") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; + return TMQ_CONF_OK; + } else if (strcmp(value, "earliest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + return TMQ_CONF_OK; + } else if (strcmp(value, "latest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } return TMQ_CONF_UNKNOWN; } @@ -190,14 +201,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } - if (!pParam->async) tsem_post(&pParam->rspSem); - return 0; -} - -int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param; - pParam->rspErr = code; - tsem_post(&pParam->rspSem); + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + free(param); + } return 0; } @@ -216,6 +225,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->auto_commit; pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); @@ -223,18 +233,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } -tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - SRequestObj* pRequest = NULL; +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { + // TODO: add read write lock + SRequestObj* pRequest = NULL; + tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS; // build msg // send to mnode - SMqCMResetOffsetReq req; - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + SMqCMCommitOffsetReq req; + SArray* pArray = NULL; + + if (offsets == NULL) { + pArray = taosArrayInit(0, sizeof(SMqOffset)); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqOffset offset; + strcpy(offset.topicName, pTopic->topicName); + strcpy(offset.cgroup, tmq->groupId); + offset.vgId = pVg->vgId; + offset.offset = pVg->currentOffset; + taosArrayPush(pArray, &offset); + } + } + req.num = pArray->size; + req.offsets = pArray->pData; + } else { + req.num = offsets->cnt; + req.offsets = (SMqOffset*)offsets->elems; + } SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; void* buf = malloc(tlen); if (buf == NULL) { @@ -244,32 +276,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse tCoderClear(&encoder); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); tCoderClear(&encoder); - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET); + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET); if (pRequest == NULL) { tscError("failed to malloc request"); } - SMqResetOffsetParam param = {0}; - tsem_init(¶m.rspSem, 0, 0); - param.tmq = tmq; + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + return -1; + } + pParam->tmq = tmq; + tsem_init(&pParam->rspSem, 0, 0); pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->param = ¶m; - sendInfo->fp = tmqResetOffsetCb; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(¶m.rspSem); - tsem_destroy(¶m.rspSem); + if (!async) { + tsem_wait(&pParam->rspSem); + resp = pParam->rspErr; + } - return param.rspErr; + if (pArray) { + taosArrayDestroy(pArray); + } + + return resp; } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { @@ -641,8 +682,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // clang-format off SMqClientVg clientVg = { .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, + .currentOffset = pVgEp->offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet }; @@ -708,23 +748,51 @@ END: return 0; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, - SMqClientVg* pVg) { +tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { + const SMqOffset* pOffset = &offset->offset; + if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { + return TMQ_RESP_ERR__FAIL; + } + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) { + int32_t vgSz = taosArrayGetSize(clientTopic->vgs); + for (int32_t j = 0; j < vgSz; j++) { + SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); + if (pVg->vgId == pOffset->vgId) { + pVg->currentOffset = pOffset->offset; + return TMQ_RESP_ERR__SUCCESS; + } + } + } + } + return TMQ_RESP_ERR__FAIL; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { + int64_t reqOffset; + if (pVg->currentOffset >= 0) { + reqOffset = pVg->currentOffset; + } else { + if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) { + tscError("unable to poll since no committed offset but reset offset is set to none"); + return NULL; + } + reqOffset = tmq->resetOffsetCfg; + } + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); if (pReq == NULL) { return NULL; } - pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; strcpy(pReq->cgroup, tmq->groupId); - if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { - pReq->offset = pVg->currentOffset; - } else { - pReq->offset = pVg->currentOffset + 1; - } + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -743,13 +811,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - printf("over1\n"); + /*printf("over1\n");*/ usleep(blocking_time * 1000); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { - printf("over2\n"); + /*printf("over2\n");*/ usleep(blocking_time * 1000); return NULL; } @@ -760,8 +828,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); if (pReq == NULL) { ASSERT(false); usleep(blocking_time * 1000); @@ -821,6 +888,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } +#if 0 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { if (tmq_topic_vgroup_list != NULL) { // TODO @@ -831,7 +899,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; @@ -858,6 +926,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v return 0; } +#endif void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c index 3c0e6df735..ef441c97c7 100644 --- a/source/common/src/tcompare.c +++ b/source/common/src/tcompare.c @@ -15,116 +15,6 @@ #include "tcompare.h" -int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); - memcpy(buf, varDataVal(pLeft), sz); - buf[sz] = 0; - - int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - free(buf); - free(pattern); - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - - wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); - free(pattern); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -__compar_fn_t getComparFunc(int32_t type, int32_t optr) { - __compar_fn_t comparFn = NULL; - - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - return setCompareBytes1; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - return setCompareBytes2; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - return setCompareBytes4; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - return setCompareBytes8; - default: - assert(0); - } - } - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; - case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; - case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternComp; - } else if (optr == TSDB_RELATION_IN) { - comparFn = compareFindItemInSet; - } else { /* normal relational comparFn */ - comparFn = compareLenPrefixedStr; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { - comparFn = compareWStrPatternComp; - } else if (optr == TSDB_RELATION_IN) { - comparFn = compareFindItemInSet; - } else { - comparFn = compareLenPrefixedWStr; - } - break; - } - - case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; - case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; - - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { __compar_fn_t comparFn = NULL; @@ -218,4 +108,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { } } } -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 46feab7791..8101c18ebf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->ver)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); @@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); @@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { uint32_t nsize = 0; - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { @@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU8(buf, pReq->type); return tlen; } void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeString(buf, &pReq->name); buf = taosDecodeFixedU8(buf, &pReq->type); return buf; @@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; @@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; @@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; @@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; @@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; @@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; @@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; tEndEncode(&encoder); @@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; tEndDecode(&decoder); @@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) { return 0; } -int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) { - if (tStartEncode(encoder) < 0) return -1; - if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1; - int32_t sz = taosArrayGetSize(pOffsets->offsets); - if (tEncodeI32(encoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SMqOffset *offset = taosArrayGet(pOffsets->offsets, i); - if (tEncodeSMqOffset(encoder, offset) < 0) return -1; - } - tEndEncode(encoder); - return encoder->pos; -} - -int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) { - int32_t sz; - if (tStartDecode(decoder) < 0) return -1; - if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1; - if (tDecodeI32(decoder, &sz) < 0) return -1; - pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset)); - for (int32_t i = 0; i < sz; i++) { - SMqOffset offset; - if (tDecodeSMqOffset(decoder, &offset) < 0) return -1; - taosArrayPush(pOffsets->offsets, &offset); - } - tEndDecode(decoder); - return 0; -} - -int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) { +int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) { if (tStartEncode(encoder) < 0) return -1; if (tEncodeI32(encoder, pReq->num) < 0) return -1; for (int32_t i = 0; i < pReq->num; i++) { @@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p return encoder->pos; } -int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { +int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); @@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { tEndDecode(decoder); return 0; } - -#if 0 -int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) { - if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tEncodeSMqOffset(encoder, &pReq->offsets[i]); - } - return encoder->pos; -} - -int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) { - if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); - if (pReq->offsets == NULL) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tDecodeSMqOffset(decoder, &pReq->offsets[i]); - } - return 0; -} -#endif diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f3deb84ccf..f6892b26bd 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil } assert(src->filterstr == 0 || src->filterstr == 1); - assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0)); return pFilter; } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 1cab413c3f..ee32a20920 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -585,7 +585,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { - if (optr == TSDB_BINARY_OP_ADD) { + if (optr == OP_TYPE_ADD) { switch (type) { case TSDB_DATA_TYPE_TINYINT: *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index a0ba71a1eb..0aae145d2f 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -25,8 +25,8 @@ #include "dndMnode.h" #include "dndVnodes.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dndInitMsgFp(STransMgmt *pMgmt) { @@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; @@ -193,6 +194,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; rpcInit.parent = pDnode; + rpcInit.noPool = true; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/impl/test/sut/inc/client.h index 9cf688fc02..925680d528 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/client.h +++ b/source/dnode/mgmt/impl/test/sut/inc/client.h @@ -21,16 +21,21 @@ class TestClient { bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port); void Cleanup(); + void DoInit(); + SRpcMsg* SendReq(SRpcMsg* pReq); void SetRpcRsp(SRpcMsg* pRsp); tsem_t* GetSem(); + void Restart(); private: char fqdn[TSDB_FQDN_LEN]; uint16_t port; + char user[128]; + char pass[128]; void* clientRpc; SRpcMsg* pRsp; tsem_t sem; }; -#endif /* _TD_TEST_CLIENT_H_ */ \ No newline at end of file +#endif /* _TD_TEST_CLIENT_H_ */ diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index c1f52844a4..5e42ee78df 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -23,6 +23,7 @@ #include "tmsg.h" #include "tdataformat.h" #include "tglobal.h" +#include "tmsg.h" #include "tnote.h" #include "trpc.h" #include "tthread.h" @@ -38,6 +39,7 @@ class Testbase { void Restart(); void ServerStop(); void ServerStart(); + void ClientRestart(); SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen); private: @@ -99,7 +101,7 @@ class Testbase { { \ char* bytes = (char*)calloc(1, len); \ for (int32_t i = 0; i < len - 1; ++i) { \ - bytes[i] = b; \ + bytes[i] = b; \ } \ EXPECT_STREQ(test.GetShowBinary(len), bytes); \ } @@ -137,4 +139,4 @@ class Testbase { #define IgnoreTimestamp() \ { test.GetShowTimestamp(); } -#endif /* _TD_TEST_BASE_H_ */ \ No newline at end of file +#endif /* _TD_TEST_BASE_H_ */ diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index 8403dbf034..b89cb02834 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -13,33 +13,41 @@ * along with this program. If not, see . */ -#include "tep.h" #include "sut.h" +#include "tep.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; client->SetRpcRsp(pRsp); - uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pRsp->msgType), pRsp->code); + uInfo("x response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen); tsem_post(client->GetSem()); } -void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; +void TestClient::SetRpcRsp(SRpcMsg* rsp) { + if (this->pRsp) { + free(this->pRsp); + } + this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg)); + this->pRsp->msgType = rsp->msgType; + this->pRsp->code = rsp->code; + this->pRsp->pCont = rsp->pCont; + this->pRsp->contLen = rsp->contLen; +}; tsem_t* TestClient::GetSem() { return &sem; } -bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { +void TestClient::DoInit() { char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); - SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = (char*)"DND-C"; + rpcInit.label = (char*)"shell"; rpcInit.numOfThreads = 1; rpcInit.cfp = processClientRsp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 30 * 1000; - rpcInit.user = (char*)user; + rpcInit.user = (char*)this->user; rpcInit.ckey = (char*)"key"; rpcInit.parent = this; rpcInit.secret = (char*)secretEncrypt; @@ -47,11 +55,16 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint clientRpc = rpcOpen(&rpcInit); ASSERT(clientRpc); + tsem_init(&this->sem, 0, 0); +} - tsem_init(&sem, 0, 0); +bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { strcpy(this->fqdn, fqdn); + strcpy(this->user, user); + strcpy(this->pass, pass); this->port = port; - + this->pRsp = NULL; + this->DoInit(); return true; } @@ -60,11 +73,16 @@ void TestClient::Cleanup() { rpcClose(clientRpc); } +void TestClient::Restart() { + this->Cleanup(); + this->DoInit(); +} SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) { SEpSet epSet = {0}; addEpIntoEpSet(&epSet, fqdn, port); rpcSendRequest(clientRpc, &epSet, pReq, NULL); tsem_wait(&sem); + uInfo("y response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen); return pRsp; } diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 30b796290f..ac199107a0 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -21,9 +21,9 @@ void Testbase::InitLog(const char* path) { mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; - tmrDebugFlag = 0; - uDebugFlag = 0; - rpcDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; qDebugFlag = 0; wDebugFlag = 0; sDebugFlag = 0; @@ -62,16 +62,21 @@ void Testbase::Init(const char* path, int16_t port) { void Testbase::Cleanup() { tFreeSTableMetaRsp(&metaRsp); - server.Stop(); client.Cleanup(); + taosMsleep(10); + server.Stop(); dndCleanup(); } -void Testbase::Restart() { server.Restart(); } +void Testbase::Restart() { + server.Restart(); + client.Restart(); +} void Testbase::ServerStop() { server.Stop(); } void Testbase::ServerStart() { server.DoStart(); } +void Testbase::ClientRestart() { client.Restart(); } SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { SRpcMsg rpcMsg = {0}; @@ -190,4 +195,4 @@ int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; } STableMetaRsp* Testbase::GetShowMeta() { return &metaRsp; } -SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } \ No newline at end of file +SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e64191f715..bb2367ef72 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -123,6 +123,7 @@ typedef enum { TRN_TYPE_DROP_TOPIC = 1015, TRN_TYPE_SUBSCRIBE = 1016, TRN_TYPE_REBALANCE = 1017, + TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -176,7 +177,7 @@ typedef struct { SArray* undoActions; int64_t createdTime; int64_t lastExecTime; - uint64_t dbUid; + int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; } STrans; @@ -304,16 +305,16 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - char createUser[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - uint64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + char createUser[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; } SDbObj; typedef struct { @@ -346,8 +347,8 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t createdTime; int64_t updateTime; - uint64_t uid; - uint64_t dbUid; + int64_t uid; + int64_t dbUid; int32_t version; int32_t nextColId; int32_t numOfColumns; @@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { } } +typedef struct { + char key[TSDB_PARTITION_KEY_LEN]; + int64_t offset; +} SMqOffsetObj; + +static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} + typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t status; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h new file mode 100644 index 0000000000..e18eec973f --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_OFFSET_H_ +#define _TD_MND_OFFSET_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitOffset(SMnode *pMnode); +void mndCleanupOffset(SMnode *pMnode); + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key); +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); + +static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { + return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); +} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_OFFSET_H_*/ diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c new file mode 100644 index 0000000000..ac9e99ebd4 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndOffset.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_OFFSET_VER_NUMBER 1 +#define MND_OFFSET_RESERVE_SIZE 64 + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); + +int32_t mndInitOffset(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupOffset(SMnode *pMnode) {} + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset); + int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); + if (pRaw == NULL) goto OFFSET_ENCODE_OVER; + + buf = malloc(tlen); + if (buf == NULL) goto OFFSET_ENCODE_OVER; + + void *abuf = buf; + tEncodeSMqOffsetObj(&abuf, pOffset); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_ENCODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRaw; +} + +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER; + + if (sver != MND_OFFSET_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto OFFSET_DECODE_OVER; + } + + int32_t size = sizeof(SMqOffsetObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto OFFSET_DECODE_OVER; + + SMqOffsetObj *pOffset = sdbGetRowObj(pRow); + if (pOffset == NULL) goto OFFSET_DECODE_OVER; + + int32_t dataPos = 0; + int32_t tlen; + SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); + buf = malloc(tlen + 1); + if (buf == NULL) goto OFFSET_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); + + if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) { + goto OFFSET_DECODE_OVER; + } + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_DECODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); + tfree(pRow); + return NULL; + } + + mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRow; +} + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { + int32_t code = 0; + int32_t sz = taosArrayGetSize(vgs); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + return -1; + } + offsetObj.offset = -1; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj); + if (pOffsetRaw == NULL) { + return -1; + } + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) { + return -1; + } + } + return 0; +} + +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { + char key[TSDB_PARTITION_KEY_LEN]; + + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SMqCMCommitOffsetReq commitOffsetReq; + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER); + + tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg); + + for (int32_t i = 0; i < commitOffsetReq.num; i++) { + SMqOffset *pOffset = &commitOffsetReq.offsets[i]; + if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { + return -1; + } + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); + ASSERT(pOffsetObj); + pOffsetObj->offset = pOffset->offset; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pOffsetRaw); + mndReleaseOffset(pMnode, pOffsetObj); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform insert action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform delete action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { + mTrace("offset:%s, perform update action", pOldOffset->key); + return 0; +} + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) { + SSdb *pSdb = pMnode->pSdb; + SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key); + if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST; + } + return pOffset; +} + +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pOffset); +} + +static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a5c4c41244..2ea157fea4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -19,6 +19,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" #include "mndTopic.h" @@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) { +static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { SMqSubscribeObj *pSub = tNewSubscribeObj(); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name); + char *key = mndMakeSubscribeKey(cgroup, pTopic->name); if (key == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); @@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); for (int32_t k = 0; k < vgsz; k++) { + char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId}; + SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = pOffsetObj->offset; + mndReleaseOffset(pMnode, pOffsetObj); + } taosArrayPush(topicEp.vgs, &vgEp); } taosArrayPush(rsp.topics, &topicEp); @@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SUB_ENCODE_OVER: tfree(buf); - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; @@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; + + mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); } SMqSubConsumer mqSubConsumer; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 4a2b61898c..299e66a7c0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -23,6 +23,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" @@ -217,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index d4e308268a..b8a0e61ca3 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -190,6 +190,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); server2.Stop(); + taosMsleep(1000); + // test.ClientRestart(); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); @@ -226,6 +229,7 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { { // server start, wait until the rollback finished server2.DoStart(); + test.ClientRestart(); taosMsleep(1000); int32_t retry = 0; @@ -248,7 +252,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { ASSERT_NE(retry, retryMax); } } - TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started @@ -315,4 +318,4 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { ASSERT_NE(retry, retryMax); } -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index d4c40dd428..88d4fc4f75 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -46,8 +46,10 @@ class MndTestTrans : public ::testing::Test { free(buffer); taosFsyncFile(fd); taosCloseFile(fd); + taosMsleep(1000); test.ServerStart(); + test.ClientRestart(); } static Testbase test; @@ -284,6 +286,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { } } + // create db // partial create stb // drop db failed diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index d8ce599be1..61b99beeb7 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -617,6 +617,7 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { // restart test.Restart(); + taosMsleep(1000); test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); @@ -631,4 +632,4 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { CheckTimestamp(); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4dc46b3798..ac9dde3597 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t fetchOffset = pReq->offset; + int64_t fetchOffset; /*int64_t blockingTime = pReq->blockingTime;*/ + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = 0; + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetLastVer(pTq->pWal); + } else { + fetchOffset = pReq->currentOffset + 1; + } + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); @@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); ASSERT(pConsumer->consumerId == consumerId); - if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { - pTopic->committedOffset = pReq->offset; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; - } - - if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - if (pTopic->committedOffset < pReq->offset - 1) { - pTopic->committedOffset = pReq->offset - 1; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - } - } - - rsp.committedOffset = pTopic->committedOffset; - rsp.reqOffset = pReq->offset; + rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - if (fetchOffset <= pTopic->committedOffset) { - fetchOffset = pTopic->committedOffset + 1; - } - SWalHead* pHead; while (1) { int8_t pos = fetchOffset % TQ_BUFFER_SIZE; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f779949f14..4b47413715 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -126,7 +126,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - metaRsp.dbId = htobe64(pVnode->config.dbId); + metaRsp.dbId = pVnode->config.dbId; memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); strcpy(metaRsp.tbName, infoReq.tbName); if (pTbCfg->type == META_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade280eecc..c3947da459 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - // ser request version + // set request version void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { - /*ASSERT(false);*/ // TODO: handle error + /*ASSERT(false);*/ + vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); } } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index f515dda4bb..da2ab5b425 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -14,4 +14,5 @@ add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) add_subdirectory(nodes) -add_subdirectory(config) \ No newline at end of file +add_subdirectory(config) +add_subdirectory(scalar) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b5c7f44f98..d62c189d33 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -546,7 +546,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); + ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -1556,39 +1556,43 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons STableMetaOutput *output = NULL; - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + while (true) { + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); - if (CTG_IS_META_TABLE(output->metaType)) { - *pTableMeta = output->tbMeta; - goto _return; - } + if (CTG_IS_META_TABLE(output->metaType)) { + *pTableMeta = output->tbMeta; + goto _return; + } - if (CTG_IS_META_BOTH(output->metaType)) { - memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + if (CTG_IS_META_BOTH(output->metaType)) { + memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + *pTableMeta = output->tbMeta; + goto _return; + } + + if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { + ctgError("invalid metaType:%d", output->metaType); + tfree(output->tbMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + // HANDLE ONLY CHILD TABLE META + + SName stbName = *pTableName; + strcpy(stbName.tname, output->tbName); - *pTableMeta = output->tbMeta; - goto _return; + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); + if (0 == exist) { + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + continue; + } + + memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + break; } - if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { - ctgError("invalid metaType:%d", output->metaType); - tfree(output->tbMeta); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - // HANDLE ONLY CHILD TABLE META - - SName stbName = *pTableName; - strcpy(stbName.tname, output->tbName); - - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); - if (0 == exist) { - ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); - CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST); - } - - memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - _return: tfree(output); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 9b53cc1fbb..12a78134c3 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode + PRIVATE os util common function parser planner qcom vnode scalar nodes ) target_include_directories( diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 232b54554f..23acdf1c33 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -29,9 +29,9 @@ extern "C" { #include "executil.h" #include "executor.h" #include "planner.h" +#include "scalar.h" #include "taosdef.h" #include "tarray.h" -#include "tfilter.h" #include "thash.h" #include "tlockfree.h" #include "tpagedbuf.h" diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 18485249b3..4078ee9291 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } else { SStreamBlockScanInfo* pInfo = pOperator->info; if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { - qError("submit msg error while set stream msg, %s" PRIx64, id); + qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f5dc7a82b1..5508b5ecd7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2148,7 +2148,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); + //pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { @@ -2174,7 +2174,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT return TSDB_CODE_SUCCESS; _clean: - destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); + //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); tfree(pRuntimeEnv->pResultRowHashTable); tfree(pRuntimeEnv->keyBuf); tfree(pRuntimeEnv->prevRow); @@ -2212,7 +2212,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); + //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); // destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt index ece84207c7..c24993eb89 100644 --- a/source/libs/executor/test/CMakeLists.txt +++ b/source/libs/executor/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( executorTest - PUBLIC os util common transport gtest taos qcom executor function planner + PRIVATE os util common transport gtest taos qcom executor function planner scalar nodes ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index ebea6755d7..c9657607c2 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -32,6 +32,8 @@ #include "trpc.h" #include "stub.h" #include "executor.h" +#include "tmsg.h" +#include "tname.h" namespace { diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index 62f86de918..598a28b2eb 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -46,6 +46,7 @@ typedef struct SBuiltinFuncDefinition { FExecGetEnv getEnvFunc; FExecInit initFunc; FExecProcess processFunc; + FScalarExecProcess sprocessFunc; FExecFinalize finalizeFunc; } SBuiltinFuncDefinition; diff --git a/source/libs/function/inc/tunaryoperator.h b/source/libs/function/inc/tunaryoperator.h index 27784e8487..cd40297e07 100644 --- a/source/libs/function/inc/tunaryoperator.h +++ b/source/libs/function/inc/tunaryoperator.h @@ -20,10 +20,10 @@ extern "C" { #endif -#include "tscalarfunction.h" +//#include "tscalarfunction.h" -typedef void (*_unary_scalar_fn_t)(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput); -_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); +//typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput); +//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 96e165f788..73ce67bd28 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -36,7 +36,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = stubCheckAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .processFunc = NULL, + .sprocessFunc = NULL, .finalizeFunc = NULL } }; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 887f65a6ea..c6330c6015 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -71,6 +71,14 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { return TSDB_CODE_SUCCESS; } +int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return TSDB_CODE_FAILED; + } + pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc; + return TSDB_CODE_SUCCESS; +} + bool fmIsAggFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index d0d89611c4..1aa5871468 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "tscalarfunction.h" #include "os.h" #include "taosdef.h" #include "tmsg.h" @@ -3221,6 +3220,7 @@ static void diff_function(SqlFunctionCtx *pCtx) { } } +#if 0 char *getArithColumnData(void *param, const char* name, int32_t colId) { SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; @@ -3235,15 +3235,16 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { assert(index >= 0); return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } +#endif static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; + //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; - SScalarFuncParam output = {0}; + SScalarParam output = {0}; output.data = pCtx->pOutput; - evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); + //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c deleted file mode 100644 index 4de11cbe71..0000000000 --- a/source/libs/function/src/tbinoperator.c +++ /dev/null @@ -1,488 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "ttypes.h" -#include "tbinoperator.h" -#include "tcompare.h" - -//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); - -void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - - *pOutput = (double)pLeft[i] + pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - - *pOutput = (double)pLeft[0] + pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - *pOutput = (double)pLeft[i] + pRight[0]; - } - } -} - -typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); - -double getVectorDoubleValue_TINYINT(void *src, int32_t index) { - return (double)*((int8_t *)src + index); -} -double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { - return (double)*((uint8_t *)src + index); -} -double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { - return (double)*((int16_t *)src + index); -} -double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { - return (double)*((uint16_t *)src + index); -} -double getVectorDoubleValue_INT(void *src, int32_t index) { - return (double)*((int32_t *)src + index); -} -double getVectorDoubleValue_UINT(void *src, int32_t index) { - return (double)*((uint32_t *)src + index); -} -double getVectorDoubleValue_BIGINT(void *src, int32_t index) { - return (double)*((int64_t *)src + index); -} -double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { - return (double)*((uint64_t *)src + index); -} -double getVectorDoubleValue_FLOAT(void *src, int32_t index) { - return (double)*((float *)src + index); -} -double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { - return (double)*((double *)src + index); -} -_getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { - _getDoubleValue_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorDoubleValue_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorDoubleValue_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorDoubleValue_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorDoubleValue_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorDoubleValue_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorDoubleValue_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorDoubleValue_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorDoubleValue_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorDoubleValue_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorDoubleValue_DOUBLE; - }else { - assert(0); - } - return p; -} - - -typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index); - -void* getVectorValueAddr_TINYINT(void *src, int32_t index) { - return (void*)((int8_t *)src + index); -} -void* getVectorValueAddr_UTINYINT(void *src, int32_t index) { - return (void*)((uint8_t *)src + index); -} -void* getVectorValueAddr_SMALLINT(void *src, int32_t index) { - return (void*)((int16_t *)src + index); -} -void* getVectorValueAddr_USMALLINT(void *src, int32_t index) { - return (void*)((uint16_t *)src + index); -} -void* getVectorValueAddr_INT(void *src, int32_t index) { - return (void*)((int32_t *)src + index); -} -void* getVectorValueAddr_UINT(void *src, int32_t index) { - return (void*)((uint32_t *)src + index); -} -void* getVectorValueAddr_BIGINT(void *src, int32_t index) { - return (void*)((int64_t *)src + index); -} -void* getVectorValueAddr_UBIGINT(void *src, int32_t index) { - return (void*)((uint64_t *)src + index); -} -void* getVectorValueAddr_FLOAT(void *src, int32_t index) { - return (void*)((float *)src + index); -} -void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { - return (void*)((double *)src + index); -} - -_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { - _getValueAddr_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorValueAddr_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorValueAddr_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorValueAddr_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorValueAddr_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorValueAddr_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorValueAddr_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorValueAddr_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorValueAddr_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorValueAddr_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorValueAddr_DOUBLE; - }else { - assert(0); - } - return p; -} - -void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); - } - } -} - -void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); - } - } -} -void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); - } - } -} - -void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); - } - } -} - -void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double * output = (double *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); - } - } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - - double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); - } - } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); - } - } -} - -void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t len = pLeft->bytes + pRight->bytes; - - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - char *output = (char *)out; - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += len) { - char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); - char* right = POINTER_SHIFT(pRight->data, pRight->bytes * i); - - if (isNull(left, pLeft->type) || isNull(right, pRight->type)) { - setVardataNull(output, TSDB_DATA_TYPE_BINARY); - continue; - } - - // todo define a macro - memcpy(varDataVal(output), varDataVal(left), varDataLen(left)); - memcpy(varDataVal(output) + varDataLen(left), varDataVal(right), varDataLen(right)); - varDataSetLen(output, varDataLen(left) + varDataLen(right)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += len) { - char *right = POINTER_SHIFT(pRight->data, pRight->bytes * i); - if (isNull(pLeft->data, pLeft->type) || isNull(right, pRight->type)) { - setVardataNull(output, TSDB_DATA_TYPE_BINARY); - continue; - } - - memcpy(varDataVal(output), varDataVal(pLeft->data), varDataLen(pLeft->data)); - memcpy(varDataVal(output) + varDataLen(pLeft->data), varDataVal(right), varDataLen(right)); - varDataSetLen(output, varDataLen(pLeft->data) + varDataLen(right)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += len) { - char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); - if (isNull(left, pLeft->type) || isNull(pRight->data, pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - memcpy(varDataVal(output), varDataVal(left), varDataLen(pRight->data)); - memcpy(varDataVal(output) + varDataLen(left), varDataVal(pRight->data), varDataLen(pRight->data)); - varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); - } - } - -} - -_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { - switch (binFunctionId) { - case TSDB_BINARY_OP_ADD: - return vectorAdd; - case TSDB_BINARY_OP_SUBTRACT: - return vectorSub; - case TSDB_BINARY_OP_MULTIPLY: - return vectorMultiply; - case TSDB_BINARY_OP_DIVIDE: - return vectorDivide; - case TSDB_BINARY_OP_REMAINDER: - return vectorRemainder; - case TSDB_BINARY_OP_CONCAT: - return vectorConcat; - default: - assert(0); - return NULL; - } -} - -bool isBinaryStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index aa3e6ba0f0..7d9cb97400 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -25,6 +25,7 @@ #include "thash.h" #include "texpr.h" #include "tvariant.h" +#include "tdef.h" //static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { // if (pLeft->nodeType == TEXPR_COL_NODE) { @@ -94,7 +95,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp //non-leaf nodes, recursively traverse the expression tree in the post-root order if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) { // or if (exprTreeApplyFilter(pLeft, pItem, param)) { return true; } @@ -255,7 +256,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_LIKE; + expr->_node.optr = OP_TYPE_LIKE; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; @@ -266,7 +267,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_MATCH; + expr->_node.optr = OP_TYPE_MATCH; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1; @@ -276,7 +277,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_NMATCH; + expr->_node.optr = OP_TYPE_NMATCH; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_NMATCH_LEN) + 1; @@ -286,7 +287,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_IN; + expr->_node.optr = OP_TYPE_IN; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY; diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 36c9e2513f..e302643c32 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -3,7 +3,6 @@ #include "function.h" #include "thash.h" #include "taggfunction.h" -#include "tscalarfunction.h" static SHashObj* functionHashTable = NULL; static SHashObj* udfHashTable = NULL; @@ -18,12 +17,14 @@ static void doInitFunctionHashTable() { taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } +/* numOfEntries = tListLen(scalarFunc); for(int32_t i = 0; i < numOfEntries; ++i) { int32_t len = (int32_t) strlen(scalarFunc[i].name); SScalarFunctionInfo* ptr = &scalarFunc[i]; taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } +*/ udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true); } diff --git a/source/libs/function/src/tunaryoperator.c b/source/libs/function/src/tunaryoperator.c index 9651f98d08..957f0799c5 100644 --- a/source/libs/function/src/tunaryoperator.c +++ b/source/libs/function/src/tunaryoperator.c @@ -4,10 +4,10 @@ // TODO dynamic define these functions -_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { - assert(0); -} +//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { +// assert(0); +//} -bool isStringOperatorFn(int32_t op) { - return op == FUNCTION_LENGTH; -} +//bool isStringOperatorFn(int32_t op) { +// return op == FUNCTION_LENGTH; +//} diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 0c222eae1a..9287a91828 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -66,7 +66,9 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -76,7 +78,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); - if (sIdx->tindex == NULL) { goto END; } + if (sIdx->tindex == NULL) { + goto END; + } sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; @@ -87,7 +91,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #endif END: - if (sIdx != NULL) { indexClose(sIdx); } + if (sIdx != NULL) { + indexClose(sIdx); + } *index = NULL; return -1; @@ -103,7 +109,9 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { indexCacheUnRef(*pCache); } + if (*pCache) { + indexCacheUnRef(*pCache); + } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); @@ -161,7 +169,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { return ret; } + if (ret != 0) { + return ret; + } } #endif @@ -191,7 +201,9 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } + for (int i = 0; i < tsz; i++) { + taosArrayPush(result, &tResult[i]); + } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -248,7 +260,9 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { return NULL; } + if (p == NULL) { + return NULL; + } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -270,7 +284,9 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { return NULL; } + if (t == NULL) { + return NULL; + } t->suid = suid; t->operType = oper; @@ -343,7 +359,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { return; } + if (results == NULL) { + return; + } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -419,18 +437,24 @@ static void indexDestroyTempResult(SArray* result) { taosArrayDestroy(result); } int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); int64_t st = taosGetTimestampUs(); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); - if (pReader == NULL) { indexWarn("empty tfile reader found"); } + if (pReader == NULL) { + indexWarn("empty tfile reader found"); + } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); - if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } + if (tfileIter == NULL) { + indexWarn("empty tfile reader iterator"); + } SArray* result = taosArrayInit(1024, sizeof(void*)); @@ -484,7 +508,9 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayDestroy(value->val); value->val = NULL; } else { - if (value->val != NULL) { taosArrayClear(value->val); } + if (value->val != NULL) { + taosArrayClear(value->val); + } } free(value->colVal); value->colVal = NULL; @@ -507,7 +533,9 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { tfileWriterClose(tw); TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); - if (reader == NULL) { return -1; } + if (reader == NULL) { + return -1; + } TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 48566a8674..d6a7141825 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -119,13 +119,17 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); + tbl = cache->imm; cache->imm = NULL; // or throw int bg thread pthread_cond_broadcast(&cache->finished); + pthread_mutex_unlock(&cache->mtx); indexMemUnRef(tbl); @@ -133,7 +137,9 @@ void indexCacheDestroyImm(IndexCache* cache) { } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { return; } + if (pCache == NULL) { + return; + } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -146,7 +152,9 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { return NULL; } + if (iiter == NULL) { + return NULL; + } pthread_mutex_lock(&cache->mtx); @@ -164,7 +172,9 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { return; } + if (iter == NULL) { + return; + } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -186,9 +196,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } else if (cache->imm != NULL) { // TODO: wake up by condition variable pthread_cond_wait(&cache->finished, &cache->mtx); - // pthread_mutex_unlock(&cache->mtx); - // taosMsleep(50); - // pthread_mutex_lock(&cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; @@ -202,13 +209,17 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { return -1; } + if (cache == NULL) { + return -1; + } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { return -1; } + if (cache == NULL) { + return -1; + } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -240,7 +251,9 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u } static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { - if (mem == NULL) { return 0; } + if (mem == NULL) { + return 0; + } char* key = indexCacheTermGet(ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); @@ -266,7 +279,9 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA return 0; } int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { return 0; } + if (cache == NULL) { + return 0; + } IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; @@ -294,23 +309,33 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } int ref = T_REF_DEC(cache); - if (ref == 0) { indexCacheDestroy(cache); } + if (ref == 0) { + indexCacheDestroy(cache); + } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { return; } + if (tbl == NULL) { + return; + } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { return; } + if (tbl == NULL) { + return; + } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -320,7 +345,9 @@ void indexMemUnRef(MemTable* tbl) { } static void indexCacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { return; } + if (ct == NULL) { + return; + } free(ct->colVal); free(ct); } @@ -333,7 +360,9 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { CacheTerm* rt = (CacheTerm*)r; // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { return rt->version - lt->version; } + if (cmp == 0) { + return rt->version - lt->version; + } return cmp; } @@ -354,7 +383,9 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { return false; } + if (iter == NULL) { + return false; + } IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 46b4c9d7c6..3664bcfad0 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -31,20 +31,24 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi FstUnFinishedNodes* fstUnFinishedNodesCreate() { FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); - if (nodes == NULL) { return NULL; } + if (nodes == NULL) { + return NULL; + } nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } -void unFinishedNodeDestroyElem(void* elem) { +static void unFinishedNodeDestroyElem(void* elem) { FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); free(b->last); b->last = NULL; } void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { - if (nodes == NULL) { return; } + if (nodes == NULL) { + return; + } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); free(nodes); @@ -92,7 +96,9 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr add } void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { FstSlice* s = &bs; - if (fstSliceIsEmpty(s)) { return; } + if (fstSliceIsEmpty(s)) { + return; + } size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); @@ -172,7 +178,9 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fs = {.state = EmptyFinal, .val = 0}; - if (addr == EMPTY_ADDRESS) { return fs; } + if (addr == EMPTY_ADDRESS) { + return fs; + } uint8_t* data = fstSliceData(slice, NULL); uint8_t v = data[addr]; @@ -229,7 +237,9 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran fstStateSetCommInput(&st, trn->inp); bool null = false; uint8_t inp = fstStateCommInput(&st, &null); - if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); } + if (null == true) { + fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); + } fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); return; } @@ -263,7 +273,9 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil fstStateSetStateNtrans(&st, (uint8_t)sz); if (anyOuts) { - if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); } + if (FST_BUILDER_NODE_IS_FINAL(node)) { + fstCountingWriterPackUintIn(w, node->finalOutput, oSize); + } for (int32_t i = sz - 1; i >= 0; i--) { FstTransition* t = taosArrayGet(node->trans, i); fstCountingWriterPackUintIn(w, t->out, oSize); @@ -428,7 +440,9 @@ Output fstStateOutput(FstState* s, FstNode* node) { assert(s->state == OneTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { return 0; } + if (oSizes == 0) { + return 0; + } FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); @@ -440,7 +454,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { return 0; } + if (oSizes == 0) { + return 0; + } FstSlice* slice = &node->data; uint8_t* data = fstSliceData(slice, NULL); uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -453,7 +469,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { void fstStateSetFinalState(FstState* s, bool yes) { assert(s->state == AnyTrans); - if (yes) { s->val |= 0b01000000; } + if (yes) { + s->val |= 0b01000000; + } return; } bool fstStateIsFinalState(FstState* s) { @@ -463,7 +481,9 @@ bool fstStateIsFinalState(FstState* s) { void fstStateSetStateNtrans(FstState* s, uint8_t n) { assert(s->state == AnyTrans); - if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } + if (n <= 0b00111111) { + s->val = (s->val & 0b11000000) | n; + } return; } // state_ntrans @@ -495,7 +515,9 @@ uint64_t fstStateNtransLen(FstState* s) { uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { bool null = false; uint8_t n = fstStateStateNtrans(s, &null); - if (null != true) { return n; } + if (null != true) { + return n; + } int32_t len; uint8_t* data = fstSliceData(slice, &len); n = data[len - 2]; @@ -505,7 +527,9 @@ uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { } Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; } + if (oSizes == 0 || !fstStateIsFinalState(s)) { + return 0; + } uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; @@ -522,7 +546,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { uint8_t* data = fstSliceData(slice, &dlen); uint64_t i = data[at + b]; // uint64_t i = slice->data[slice->start + at + b]; - if (i >= node->nTrans) { *null = true; } + if (i >= node->nTrans) { + *null = true; + } return i; } else { uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size @@ -539,7 +565,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { return node->nTrans - i - 1; // bug } } - if (i == len) { *null = true; } + if (i == len) { + *null = true; + } fstSliceDestroy(&t); } } @@ -548,7 +576,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { FstNode* n = (FstNode*)malloc(sizeof(FstNode)); - if (n == NULL) { return NULL; } + if (n == NULL) { + return NULL; + } FstState st = fstStateCreateFrom(slice, addr); @@ -614,7 +644,9 @@ void fstNodeDestroy(FstNode* node) { } FstTransitions* fstNodeTransitions(FstNode* node) { FstTransitions* t = malloc(sizeof(FstTransitions)); - if (NULL == t) { return NULL; } + if (NULL == t) { + return NULL; + } FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; t->range = range; t->node = node; @@ -721,7 +753,9 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled FstBuilder* fstBuilderCreate(void* w, FstType ty) { FstBuilder* b = malloc(sizeof(FstBuilder)); - if (NULL == b) { return b; } + if (NULL == b) { + return b; + } b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); @@ -735,15 +769,17 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) { taosEncodeFixedU64(&pBuf64, VERSION); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - memset(buf64, 0, sizeof(buf64)); pBuf64 = buf64; + memset(buf64, 0, sizeof(buf64)); taosEncodeFixedU64(&pBuf64, ty); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); return b; } void fstBuilderDestroy(FstBuilder* b) { - if (b == NULL) { return; } + if (b == NULL) { + return; + } fstCountingWriterDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); @@ -830,6 +866,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; } + CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { return EMPTY_ADDRESS; @@ -844,7 +881,9 @@ CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); - if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } + if (entry->state == NOTFOUND) { + FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); + } fstRegistryEntryDestroy(entry); return b->lastAddr; @@ -887,7 +926,9 @@ FstSlice fstNodeAsSlice(FstNode* node) { FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition* trn = malloc(sizeof(FstLastTransition)); - if (trn == NULL) { return NULL; } + if (trn == NULL) { + return NULL; + } trn->inp = inp; trn->out = out; @@ -897,7 +938,9 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; - if (trn == NULL) { return; } + if (trn == NULL) { + return; + } FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; taosArrayPush(unNode->node->trans, &t); fstLastTransitionDestroy(trn); @@ -906,27 +949,35 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp } void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { - if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; } + if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { + unNode->node->finalOutput += out; + } size_t sz = taosArrayGetSize(unNode->node->trans); for (size_t i = 0; i < sz; i++) { FstTransition* trn = taosArrayGet(unNode->node->trans, i); trn->out += out; } - if (unNode->last) { unNode->last->out += out; } + if (unNode->last) { + unNode->last->out += out; + } return; } Fst* fstCreate(FstSlice* slice) { int32_t slen; char* buf = fstSliceData(slice, &slen); - if (slen < 36) { return NULL; } + if (slen < 36) { + return NULL; + } uint64_t len = slen; uint64_t skip = 0; uint64_t version; taosDecodeFixedU64(buf, &version); skip += sizeof(version); - if (version == 0 || version > VERSION) { return NULL; } + if (version == 0 || version > VERSION) { + return NULL; + } uint64_t type; taosDecodeFixedU64(buf + skip, &type); @@ -949,10 +1000,14 @@ Fst* fstCreate(FstSlice* slice) { taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) Fst* fst = (Fst*)calloc(1, sizeof(Fst)); - if (fst == NULL) { return NULL; } + if (fst == NULL) { + return NULL; + } fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); - if (NULL == fst->meta) { goto FST_CREAT_FAILED; } + if (NULL == fst->meta) { + goto FST_CREAT_FAILED; + } fst->meta->version = version; fst->meta->rootAddr = rootAddr; @@ -983,7 +1038,7 @@ void fstDestroy(Fst* fst) { bool fstGet(Fst* fst, FstSlice* b, Output* out) { // dec lock range - pthread_mutex_lock(&fst->mtx); + // pthread_mutex_lock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; @@ -996,7 +1051,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - pthread_mutex_unlock(&fst->mtx); + // pthread_mutex_unlock(&fst->mtx); return false; } @@ -1007,7 +1062,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { - pthread_mutex_unlock(&fst->mtx); + // pthread_mutex_unlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1018,8 +1073,8 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { fstNodeDestroy(*node); } taosArrayDestroy(nodes); - fst->root = NULL; - pthread_mutex_unlock(&fst->mtx); + // fst->root = NULL; + // pthread_mutex_unlock(&fst->mtx); *out = tOut; return true; } @@ -1028,7 +1083,9 @@ FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { return fstStreamBuilderCreate(fst, ctx); } StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { - if (sb == NULL) { return NULL; } + if (sb == NULL) { + return NULL; + } return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { @@ -1039,15 +1096,6 @@ FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { FstNode* fstGetRoot(Fst* fst) { CompiledAddr rAddr = fstGetRootAddr(fst); return fstGetNode(fst, rAddr); - // pthread_mutex_lock(&fst->mtx); - // if (fst->root != NULL) { - // // pthread_mutex_unlock(&fst->mtx); - // return fst->root; - //} - // CompiledAddr rAddr = fstGetRootAddr(fst); - // fst->root = fstGetNode(fst, rAddr); - //// pthread_mutex_unlock(&fst->mtx); - // return fst->root; } FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { @@ -1074,14 +1122,18 @@ bool fstVerify(Fst* fst) { uint32_t len, checkSum = fst->meta->checkSum; uint8_t* data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data, len)) { return false; } + if (!taosCheckChecksumWhole(data, len)) { + return false; + } return true; } // data bound function FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); - if (b == NULL) { return NULL; } + if (b == NULL) { + return NULL; + } if (data != NULL) { b->data = fstSliceCopy(data, data->start, data->end); @@ -1118,7 +1170,9 @@ void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { StreamWithState* sws = calloc(1, sizeof(StreamWithState)); - if (sws == NULL) { return NULL; } + if (sws == NULL) { + return NULL; + } sws->fst = fst; sws->aut = automation; @@ -1134,7 +1188,9 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB return sws; } void streamWithStateDestroy(StreamWithState* sws) { - if (sws == NULL) { return; } + if (sws == NULL) { + return; + } taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); @@ -1200,7 +1256,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { uint64_t i = 0; for (i = trans->range.start; i < trans->range.end; i++) { FstTransition trn; - if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; } + if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { + break; + } } StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; @@ -1248,7 +1306,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb while (taosArrayGetSize(sws->stack) > 0) { StreamState* p = (StreamState*)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { - if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } + if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { + taosArrayPop(sws->inp); + } streamStateDestroy(p); continue; } @@ -1267,7 +1327,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb if (FST_NODE_IS_FINAL(nextNode)) { // void *eofState = sws->aut->acceptEof(nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); - if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } + if (eofState != NULL) { + isMatch = automFuncs[aut->type].isMatch(aut, eofState); + } } StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; taosArrayPush(sws->stack, &s1); @@ -1277,24 +1339,26 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb size_t isz = taosArrayGetSize(sws->inp); uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); - for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } + for (uint32_t i = 0; i < isz; i++) { + buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); + } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); - free(buf); + tfree(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); - free(buf); + tfree(buf); fstSliceDestroy(&slice); taosArrayDestroy(nodes); return result; } - free(buf); + tfree(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { @@ -1307,16 +1371,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); - if (result == NULL) { return NULL; } + if (result == NULL) { + return NULL; + } result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->out = fOut; result->state = state; - return result; } void swsResultDestroy(StreamWithStateResult* result) { - if (NULL == result) { return; } + if (NULL == result) { + return; + } fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); @@ -1324,16 +1391,18 @@ void swsResultDestroy(StreamWithStateResult* result) { } void streamStateDestroy(void* s) { - if (NULL == s) { return; } + if (NULL == s) { + return; + } StreamState* ss = (StreamState*)s; - fstNodeDestroy(ss->node); - // free(s->autoState); } FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); - if (NULL == b) { return NULL; } + if (NULL == b) { + return NULL; + } b->fst = fst; b->aut = aut; @@ -1349,8 +1418,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) { free(b); } FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { - if (b == NULL) { return NULL; } - + if (b == NULL) { + return NULL; + } if (type == GE) { b->min->type = Included; fstSliceDestroy(&(b->min->data)); diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index c6e3cee3e3..590ff294bf 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -17,7 +17,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = kind; nsv->type = ty; @@ -35,7 +37,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; - if (sv == NULL) { return; } + if (sv == NULL) { + return; + } if (sv->type == FST_INT) { // @@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) { } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = sv->kind; nsv->type = sv->type; @@ -88,10 +94,14 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { StartWithStateValue* ssv = (StartWithStateValue*)state; - if (ssv == NULL || ctx == NULL) { return NULL; } + if (ssv == NULL || ctx == NULL) { + return NULL; + } char* data = ctx->data; - if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; @@ -128,7 +138,9 @@ AutomationFunc automFuncs[] = { AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { return NULL; } + if (ctx == NULL) { + return NULL; + } StartWithStateValue* sv = NULL; if (atype == AUTOMATION_ALWAYS) { diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index da1e177a18..f08a48c34e 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -29,18 +29,6 @@ const uint64_t VERSION = 3; const uint64_t TRANS_INDEX_THRESHOLD = 32; -// uint8_t commonInput(uint8_t idx) { -// if (idx == 0) { return -1; } -// else { -// return COMMON_INPUTS_INV[idx - 1]; -// } -//} -// -// uint8_t commonIdx(uint8_t v, uint8_t max) { -// uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256; -// return v > max ? 0: v; -//} - uint8_t packSize(uint64_t n) { if (n < (1u << 8)) { return 1; @@ -103,9 +91,6 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) { FstString* str = s->str; str->ref++; - // uint8_t *buf = fstSliceData(s, &alen); - // start = buf + start - (buf - s->start); - // end = buf + end - (buf - s->start); FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; @@ -130,19 +115,19 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { ans.end = tlen - 1; return ans; } -bool fstSliceIsEmpty(FstSlice* s) { - return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; -} +bool fstSliceIsEmpty(FstSlice* s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } uint8_t* fstSliceData(FstSlice* s, int32_t* size) { FstString* str = s->str; - if (size != NULL) { *size = s->end - s->start + 1; } + if (size != NULL) { + *size = s->end - s->start + 1; + } return str->data + s->start; } void fstSliceDestroy(FstSlice* s) { FstString* str = s->str; str->ref--; - if (str->ref <= 0) { + if (str->ref == 0) { free(str->data); free(str); s->str = NULL; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 98fede4f7b..aff976e52b 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -13,8 +13,6 @@ p * * along with this program. If not, see . */ -//#include -//#include #include "index_tfile.h" #include "index.h" #include "index_fst.h" @@ -61,7 +59,9 @@ static void tfileGenFileFullName(char* fullname, const char* path, uint64_t s TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { return NULL; } + if (tcache == NULL) { + return NULL; + } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; @@ -98,7 +98,9 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { return; } + if (tcache == NULL) { + return; + } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -119,7 +121,9 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { int32_t sz = indexSerialCacheKey(key, buf); assert(sz < sizeof(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } tfileReaderRef(*reader); return *reader; @@ -142,7 +146,9 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } reader->ctx = ctx; @@ -169,7 +175,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); @@ -209,7 +217,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); // indexInfo("open write file name %s", fullname); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); - if (wcx == NULL) { return NULL; } + if (wcx == NULL) { + return NULL; + } TFileHeader tfh = {0}; tfh.suid = suid; @@ -225,7 +235,9 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); - if (wc == NULL) { return NULL; } + if (wc == NULL) { + return NULL; + } TFileReader* reader = tfileReaderCreate(wc); return reader; @@ -316,19 +328,25 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { return 0; } void tfileWriterClose(TFileWriter* tw) { - if (tw == NULL) { return; } + if (tw == NULL) { + return; + } writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { return; } + if (tw == NULL) { + return; + } writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { TFileCache* cache = tfileCacheCreate(path); - if (cache == NULL) { return NULL; } + if (cache == NULL) { + return NULL; + } IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); if (tfile == NULL) { @@ -340,21 +358,27 @@ IndexTFile* indexTFileCreate(const char* path) { return tfile; } void indexTFileDestroy(IndexTFile* tfile) { - if (tfile == NULL) { return; } + if (tfile == NULL) { + return; + } tfileCacheDestroy(tfile->cache); free(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { return ret; } + if (tfile == NULL) { + return ret; + } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { return 0; } + if (reader == NULL) { + return 0; + } return tfileReaderSearch(reader, query, result); } @@ -373,7 +397,9 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { return false; } + if (rt == NULL) { + return false; + } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -383,7 +409,9 @@ static bool tfileIteratorNext(Iterate* iiter) { offset = (uint64_t)(rt->out.out); swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { + return false; + } iv->colVal = colVal; return true; @@ -394,7 +422,9 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { return NULL; } + if (tIter == NULL) { + return NULL; + } tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); @@ -404,7 +434,9 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); @@ -419,7 +451,9 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { return; } + if (iter == NULL) { + return; + } IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -434,7 +468,9 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { - if (tf == NULL) { return NULL; } + if (tf == NULL) { + return NULL; + } ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } @@ -446,7 +482,9 @@ static int tfileUidCompare(const void* a, const void* b) { } static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { return ret; } + if (ret == 0) { + return ret; + } return ret < 0 ? -1 : 1; } @@ -461,13 +499,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { return NULL; } + if (tf == NULL) { + return NULL; + } tf->colVal = tstrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { return -1; } + if (tf == NULL) { + return -1; + } taosArrayPush(tf->tableId, &val); return 0; } @@ -489,7 +531,9 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { + return -1; + } indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx)); tw->offset += sizeof(fstOffset); return 0; @@ -502,7 +546,9 @@ static int tfileWriteHeader(TFileWriter* writer) { indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { return -1; } + if (sizeof(buf) != nwrite) { + return -1; + } indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx)); writer->offset = nwrite; @@ -552,23 +598,23 @@ static int tfileReaderLoadHeader(TFileReader* reader) { return 0; } static int tfileReaderLoadFst(TFileReader* reader) { - // current load fst into memory, refactor it later - static int FST_MAX_SIZE = 64 * 1024 * 1024; - - char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { return -1; } - WriterCtx* ctx = reader->ctx; int size = ctx->size(ctx); + // current load fst into memory, refactor it later + int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber); + char* buf = calloc(1, fstSize); + if (buf == NULL) { + return -1; + } + int64_t ts = taosGetTimestampUs(); - int32_t nread = - ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset); + int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset); int64_t cost = taosGetTimestampUs() - ts; - indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, - reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost); + indexInfo("nread = %d, and fst offset=%d, size: %d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, + reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost); // we assuse fst size less than FST_MAX_SIZE - assert(nread > 0 && nread < FST_MAX_SIZE); + assert(nread > 0 && nread <= fstSize); FstSlice st = fstSliceCreate((uint8_t*)buf, nread); reader->fst = fstCreate(&st); @@ -578,21 +624,35 @@ static int tfileReaderLoadFst(TFileReader* reader) { return reader->fst != NULL ? 0 : -1; } static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { - int32_t nid; + // TODO(yihao): opt later WriterCtx* ctx = reader->ctx; + char block[1024] = {0}; + int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset); + assert(nread >= sizeof(uint32_t)); - int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); - assert(sizeof(nid) == nread); + char* p = block; + int32_t nid = *(int32_t*)p; + p += sizeof(nid); - int32_t total = sizeof(uint64_t) * nid; - char* buf = calloc(1, total); - if (buf == NULL) { return -1; } + while (nid > 0) { + int32_t left = block + sizeof(block) - p; + if (left >= sizeof(uint64_t)) { + taosArrayPush(result, (uint64_t*)p); + p += sizeof(uint64_t); + } else { + char buf[sizeof(uint64_t)] = {0}; + memcpy(buf, p, left); - nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); - assert(total == nread); + memset(block, 0, sizeof(block)); + offset += sizeof(block); + nread = ctx->readFrom(ctx, block, sizeof(block), offset); + memcpy(buf + left, block, sizeof(uint64_t) - left); - for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } - free(buf); + taosArrayPush(result, (uint64_t*)buf); + p = block + sizeof(uint64_t) - left; + } + nid -= 1; + } return 0; } static int tfileReaderVerify(TFileReader* reader) { @@ -615,13 +675,17 @@ static int tfileReaderVerify(TFileReader* reader) { } void tfileReaderRef(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } int ref = T_REF_DEC(reader); if (ref == 0) { // do nothing @@ -630,18 +694,21 @@ void tfileReaderUnRef(TFileReader* reader) { } static SArray* tfileGetFileList(const char* path) { - SArray* files = taosArrayInit(4, sizeof(void*)); - char buf[128] = {0}; uint64_t suid; uint32_t version; + SArray* files = taosArrayInit(4, sizeof(void*)); DIR* dir = opendir(path); - if (NULL == dir) { return NULL; } + if (NULL == dir) { + return NULL; + } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { char* file = entry->d_name; - if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } + if (0 != tfileParseFileName(file, &suid, buf, &version)) { + continue; + } size_t len = strlen(path) + 1 + strlen(file) + 1; char* buf = calloc(1, len); diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 3957554748..665dfd7318 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,5 +1,7 @@ add_executable(indexTest "") add_executable(fstTest "") +add_executable(fstUT "") + target_sources(indexTest PRIVATE "indexTests.cc" @@ -8,6 +10,11 @@ target_sources(fstTest PRIVATE "fstTest.cc" ) + +target_sources(fstUT + PRIVATE + "fstUT.cc" +) target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -18,6 +25,12 @@ target_include_directories ( fstTest "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories ( fstUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -32,6 +45,13 @@ target_link_libraries (fstTest gtest_main index ) +target_link_libraries (fstUT + os + util + common + gtest_main + index +) #add_test( diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index a2c0046f9a..65118a2bce 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -58,7 +58,9 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { return false; } + if (nRead <= 0) { + return false; + } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -97,7 +99,8 @@ class FstReadMemory { printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); swsResultDestroy(rt); } - for (size_t i = 0; i < result.size(); i++) {} + for (size_t i = 0; i < result.size(); i++) { + } std::cout << std::endl; return true; } @@ -173,7 +176,9 @@ void checkMillonWriteAndReadOfFst() { delete fw; FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); - if (fr->init()) { printf("success to init fst read"); } + if (fr->init()) { + printf("success to init fst read"); + } Performance_fstReadRecords(fr); tfCleanup(); diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc new file mode 100644 index 0000000000..d10f5f47b8 --- /dev/null +++ b/source/libs/index/test/fstUT.cc @@ -0,0 +1,233 @@ + +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" +#include "ulog.h" + +static std::string dir = "/tmp/index"; + +static char indexlog[PATH_MAX] = {0}; +static char tindex[PATH_MAX] = {0}; +static char tindexDir[PATH_MAX] = {0}; + +static void EnvInit() { + tfInit(); + + std::string path = dir; + taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + // init log file + snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str()); + if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) { + printf("failed to init log"); + } + // init index file + memset(tindex, 0, sizeof(tindex)); + snprintf(tindex, PATH_MAX, "%s/tindex.idx", path.c_str()); +} +static void EnvCleanup() {} +class FstWriter { + public: + FstWriter() { + _wc = writerCtxCreate(TFile, tindex, false, 64 * 1024 * 1024); + _b = fstBuilderCreate(_wc, 0); + } + bool Put(const std::string& key, uint64_t val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstBuilderInsert(_b, skey, val); + + fstSliceDestroy(&skey); + return ok; + } + ~FstWriter() { + fstBuilderFinish(_b); + fstBuilderDestroy(_b); + + writerCtxDestroy(_wc, false); + } + + private: + FstBuilder* _b; + WriterCtx* _wc; +}; + +class FstReadMemory { + public: + FstReadMemory(size_t size) { + _wc = writerCtxCreate(TFile, tindex, true, 64 * 1024); + _w = fstCountingWriterCreate(_wc); + _size = size; + memset((void*)&_s, 0, sizeof(_s)); + } + bool init() { + char* buf = (char*)calloc(1, sizeof(char) * _size); + int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); + if (nRead <= 0) { + return false; + } + _size = nRead; + _s = fstSliceCreate((uint8_t*)buf, _size); + _fst = fstCreate(&_s); + free(buf); + return _fst != NULL; + } + bool Get(const std::string& key, uint64_t* val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstGet(_fst, &skey, val); + fstSliceDestroy(&skey); + return ok; + } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Get(key, val); + int64_t e = taosGetTimestampUs(); + *elapse = e - s; + return ok; + } + // add later + bool Search(AutomationCtx* ctx, std::vector& result) { + FstStreamBuilder* sb = fstSearch(_fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + // result.push_back((uint64_t)(rt->out.out)); + FstSlice* s = &rt->data; + int32_t sz = 0; + char* ch = (char*)fstSliceData(s, &sz); + std::string key(ch, sz); + printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); + swsResultDestroy(rt); + } + for (size_t i = 0; i < result.size(); i++) { + } + std::cout << std::endl; + return true; + } + bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Search(ctx, result); + int64_t e = taosGetTimestampUs(); + return ok; + } + + ~FstReadMemory() { + fstCountingWriterDestroy(_w); + fstDestroy(_fst); + fstSliceDestroy(&_s); + writerCtxDestroy(_wc, false); + tfCleanup(); + } + + private: + FstCountingWriter* _w; + Fst* _fst; + FstSlice _s; + WriterCtx* _wc; + size_t _size; +}; + +class FstWriterEnv : public ::testing::Test { + protected: + virtual void SetUp() { fw = new FstWriter(); } + virtual void TearDown() { delete fw; } + FstWriter* fw = NULL; +}; + +class FstReadEnv : public ::testing::Test { + protected: + virtual void SetUp() { fr = new FstReadMemory(1024); } + virtual void TearDown() { delete fr; } + FstReadMemory* fr = NULL; +}; + +class TFst { + public: + void CreateWriter() { fw = new FstWriter; } + void ReCreateWriter() { + if (fw != NULL) delete fw; + fw = new FstWriter; + } + void DestroyWriter() { + if (fw != NULL) delete fw; + } + void CreateReader() { + fr = new FstReadMemory(1024); + fr->init(); + } + void ReCreateReader() { + if (fr != NULL) delete fr; + fr = new FstReadMemory(1024); + } + void DestroyReader() { + delete fr; + fr = NULL; + } + bool Put(const std::string& k, uint64_t v) { + if (fw == NULL) { + return false; + } + return fw->Put(k, v); + } + bool Get(const std::string& k, uint64_t* v) { + if (fr == NULL) { + return false; + } + return fr->Get(k, v); + } + + private: + FstWriter* fw; + FstReadMemory* fr; +}; +class FstEnv : public ::testing::Test { + protected: + virtual void SetUp() { + EnvInit(); + fst = new TFst; + } + virtual void TearDown() { delete fst; } + TFst* fst; +}; + +TEST_F(FstEnv, writeNormal) { + fst->CreateWriter(); + std::string str("aa"); + for (int i = 0; i < 10; i++) { + str[0] = 'a' + i; + str.resize(2); + assert(fst->Put(str, i) == true); + } + // order failed + assert(fst->Put("aa", 1) == false); + + fst->DestroyWriter(); + + fst->CreateReader(); + uint64_t val; + assert(fst->Get("a", &val) == false); + assert(fst->Get("aa", &val) == true); + assert(val == 0); +} +TEST_F(FstEnv, writeExcpet) {} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 63bb11a821..18a316320e 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -15,6 +15,7 @@ #include "querynodes.h" #include "taos.h" +#include "taoserror.h" #define COPY_SCALAR_FIELD(fldname) \ do { \ @@ -57,14 +58,13 @@ static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) { } static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { - COPY_SCALAR_FIELD(type); dataTypeCopy(&pSrc->resType, &pDst->resType); COPY_CHAR_ARRAY_FIELD(aliasName); // COPY_NODE_LIST_FIELD(pAssociationList); } static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_SCALAR_FIELD(colId); COPY_SCALAR_FIELD(colType); COPY_CHAR_ARRAY_FIELD(dbName); @@ -76,7 +76,7 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); COPY_SCALAR_FIELD(isDuration); switch (pSrc->node.resType.type) { @@ -119,7 +119,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { } static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_SCALAR_FIELD(opType); COPY_NODE_FIELD(pLeft); COPY_NODE_FIELD(pRight); @@ -127,14 +127,14 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { } static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_SCALAR_FIELD(condType); COPY_NODE_LIST_FIELD(pParameterList); return (SNode*)pDst; } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_ARRAY_FIELD(functionName); COPY_SCALAR_FIELD(funcId); COPY_SCALAR_FIELD(funcType); @@ -142,12 +142,34 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { return (SNode*)pDst; } +static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) { + dataTypeCopy(&pSrc->dataType, &pDst->dataType); + COPY_SCALAR_FIELD(tupleId); + COPY_SCALAR_FIELD(slotId); + COPY_SCALAR_FIELD(columnId); + return (SNode*)pDst; +} + +static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { + COPY_SCALAR_FIELD(tupleId); + COPY_SCALAR_FIELD(slotId); + COPY_NODE_FIELD(pExpr); + return (SNode*)pDst; +} + +static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) { + COPY_SCALAR_FIELD(groupingSetType); + COPY_NODE_LIST_FIELD(pParameterList); + return (SNode*)pDst; +} + SNode* nodesCloneNode(const SNode* pNode) { if (NULL == pNode) { return NULL; } SNode* pDst = nodesMakeNode(nodeType(pNode)); if (NULL == pDst) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } switch (nodeType(pNode)) { @@ -161,10 +183,16 @@ SNode* nodesCloneNode(const SNode* pNode) { return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); case QUERY_NODE_FUNCTION: return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); + case QUERY_NODE_COLUMN_REF: + return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst); + case QUERY_NODE_TARGET: + return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst); case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: + break; case QUERY_NODE_GROUPING_SET: + return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst); case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_LIMIT: default: @@ -174,8 +202,13 @@ SNode* nodesCloneNode(const SNode* pNode) { } SNodeList* nodesCloneList(const SNodeList* pList) { + if (NULL == pList) { + return NULL; + } + SNodeList* pDst = nodesMakeList(); if (NULL == pDst) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SNode* pNode; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d1f32ae314..356deb2f51 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -26,41 +26,71 @@ static char* nodeName(ENodeType type) { case QUERY_NODE_COLUMN: return "Column"; case QUERY_NODE_VALUE: + return "Value"; case QUERY_NODE_OPERATOR: + return "Operator"; case QUERY_NODE_LOGIC_CONDITION: + return "LogicCondition"; case QUERY_NODE_FUNCTION: + return "Function"; case QUERY_NODE_REAL_TABLE: + return "RealTable"; case QUERY_NODE_TEMP_TABLE: + return "TempTable"; case QUERY_NODE_JOIN_TABLE: + return "JoinTable"; case QUERY_NODE_GROUPING_SET: + return "GroupingSet"; case QUERY_NODE_ORDER_BY_EXPR: + return "OrderByExpr"; case QUERY_NODE_LIMIT: + return "Limit"; case QUERY_NODE_STATE_WINDOW: + return "StateWindow"; case QUERY_NODE_SESSION_WINDOW: + return "SessionWinow"; case QUERY_NODE_INTERVAL_WINDOW: + return "IntervalWindow"; case QUERY_NODE_NODE_LIST: + return "NodeList"; case QUERY_NODE_FILL: + return "Fill"; case QUERY_NODE_COLUMN_REF: + return "ColumnRef"; case QUERY_NODE_TARGET: + return "Target"; case QUERY_NODE_RAW_EXPR: + return "RawExpr"; + case QUERY_NODE_TUPLE_DESC: + return "TupleDesc"; + case QUERY_NODE_SLOT_DESC: + return "SlotDesc"; case QUERY_NODE_SET_OPERATOR: + return "SetOperator"; case QUERY_NODE_SELECT_STMT: + return "SelectStmt"; case QUERY_NODE_SHOW_STMT: - break; + return "ShowStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: return "LogicJoin"; - case QUERY_NODE_LOGIC_PLAN_FILTER: - return "LogicFilter"; case QUERY_NODE_LOGIC_PLAN_AGG: return "LogicAgg"; case QUERY_NODE_LOGIC_PLAN_PROJECT: return "LogicProject"; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return "PhysiTagScan"; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return "PhysiTableScan"; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return "PhysiProject"; default: break; } - return "Unknown"; + static char tmp[20]; + snprintf(tmp, sizeof(tmp), "Unknown %d", type); + return tmp; } static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const SNodeList* pList) { @@ -119,7 +149,7 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; -static int32_t logicScanToJson(const void* pObj, SJson* pJson) { +static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -135,7 +165,7 @@ static int32_t logicScanToJson(const void* pObj, SJson* pJson) { static const char* jkProjectLogicPlanProjections = "Projections"; -static int32_t logicProjectToJson(const void* pObj, SJson* pJson) { +static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -149,7 +179,7 @@ static int32_t logicProjectToJson(const void* pObj, SJson* pJson) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; -static int32_t logicJoinToJson(const void* pObj, SJson* pJson) { +static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -163,14 +193,99 @@ static int32_t logicJoinToJson(const void* pObj, SJson* pJson) { return code; } -static int32_t logicFilterToJson(const void* pObj, SJson* pJson) { - return logicPlanNodeToJson(pObj, pJson); +static const char* jkPhysiPlanOutputTuple = "OutputTuple"; +static const char* jkPhysiPlanConditions = "Conditions"; +static const char* jkPhysiPlanChildren = "Children"; + +static int32_t physicPlanNodeToJson(const void* pObj, SJson* pJson) { + const SPhysiNode* pNode = (const SPhysiNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputTuple, nodeToJson, &pNode->outputTuple); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkPhysiPlanConditions, nodeToJson, pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkPhysiPlanChildren, nodeToJson, pNode->pChildren); + } + + return code; +} + +static const char* jkScanPhysiPlanScanCols = "ScanCols"; +static const char* jkScanPhysiPlanTableId = "TableId"; +static const char* jkScanPhysiPlanTableType = "TableType"; +static const char* jkScanPhysiPlanScanOrder = "ScanOrder"; +static const char* jkScanPhysiPlanScanCount = "ScanCount"; +static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount"; + +static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { + const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkScanPhysiPlanScanCols, nodeToJson, pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse); + } + + return code; +} + +static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) { + return physiScanNodeToJson(pObj, pJson); +} + +static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag"; +static const char* jkTableScanPhysiPlanStartKey = "StartKey"; +static const char* jkTableScanPhysiPlanEndKey = "EndKey"; + +static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { + const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; + + int32_t code = physiScanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanEndKey, pNode->scanRange.ekey); + } + + return code; +} + +static const char* jkProjectPhysiPlanProjections = "Projections"; + +static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { + const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkProjectPhysiPlanProjections, nodeToJson, pNode->pProjections); + } + + return code; } static const char* jkAggLogicPlanGroupKeys = "GroupKeys"; static const char* jkAggLogicPlanAggFuncs = "AggFuncs"; -static int32_t logicAggToJson(const void* pObj, SJson* pJson) { +static int32_t logicAggNodeToJson(const void* pObj, SJson* pJson) { const SAggLogicNode* pNode = (const SAggLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -184,17 +299,346 @@ static int32_t logicAggToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkDataTypeType = "Type"; +static const char* jkDataTypePrecision = "Precision"; +static const char* jkDataTypeScale = "Scale"; +static const char* jkDataTypeDataBytes = "Bytes"; + +static int32_t dataTypeToJson(const void* pObj, SJson* pJson) { + const SDataType* pNode = (const SDataType*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkDataTypeType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDataTypePrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDataTypeScale, pNode->scale); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDataTypeDataBytes, pNode->bytes); + } + + return code; +} + +static const char* jkExprDataType = "DataType"; +static const char* jkExprAliasName = "AliasName"; + +static int32_t exprNodeToJson(const void* pObj, SJson* pJson) { + const SExprNode* pNode = (const SExprNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkExprDataType, dataTypeToJson, &pNode->resType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkExprAliasName, pNode->aliasName); + } + + return code; +} + +static const char* jkColumnTableId = "TableId"; +static const char* jkColumnColId = "ColId"; +static const char* jkColumnColType = "ColType"; +static const char* jkColumnDbName = "DbName"; +static const char* jkColumnTableName = "TableName"; +static const char* jkColumnTableAlias = "TableAlias"; +static const char* jkColumnColName = "ColName"; + +static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { + const SColumnNode* pNode = (const SColumnNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnTableId, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnColType, pNode->colType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnTableAlias, pNode->tableAlias); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnColName, pNode->colName); + } + + return code; +} + +static const char* jkValueLiteral = "Literal"; +static const char* jkValueDuration = "Duration"; +static const char* jkValueDatum = "Datum"; + +static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { + const SValueNode* pNode = (const SValueNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->isDuration); + } + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.i); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.u); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + code = tjsonAddDoubleToObject(pJson, jkValueDuration, pNode->datum.d); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->datum.p); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + +static const char* jkOperatorType = "OpType"; +static const char* jkOperatorLeft = "Left"; +static const char* jkOperatorRight = "Right"; + +static int32_t operatorNodeToJson(const void* pObj, SJson* pJson) { + const SOperatorNode* pNode = (const SOperatorNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkOperatorType, pNode->opType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkOperatorLeft, nodeToJson, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkOperatorRight, nodeToJson, pNode->pRight); + } + + return code; +} + +static const char* jkLogicCondType = "CondType"; +static const char* jkLogicCondParameters = "Parameters"; + +static int32_t logicConditionNodeToJson(const void* pObj, SJson* pJson) { + const SLogicConditionNode* pNode = (const SLogicConditionNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicCondType, pNode->condType); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkLogicCondParameters, nodeToJson, pNode->pParameterList); + } + + return code; +} + +static const char* jkFunctionName = "Name"; +static const char* jkFunctionId = "Id"; +static const char* jkFunctionType = "Type"; +static const char* jkFunctionParameter = "Parameters"; + +static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { + const SFunctionNode* pNode = (const SFunctionNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkFunctionName, pNode->functionName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionId, pNode->funcId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionType, pNode->funcType); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkFunctionParameter, nodeToJson, pNode->pParameterList); + } + + return code; +} + +static const char* jkGroupingSetType = "GroupingSetType"; +static const char* jkGroupingSetParameter = "Parameters"; + +static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { + const SGroupingSetNode* pNode = (const SGroupingSetNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkGroupingSetType, pNode->groupingSetType); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkGroupingSetParameter, nodeToJson, pNode->pParameterList); + } + + return code; +} + +static const char* jkColumnRefDataType = "DataType"; +static const char* jkColumnRefTupleId = "TupleId"; +static const char* jkColumnRefSlotId = "SlotId"; +static const char* jkColumnRefColumnId = "ColumnId"; + +static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) { + const SColumnRefNode* pNode = (const SColumnRefNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId); + } + + return code; +} + +static const char* jkTargetTupleId = "TupleId"; +static const char* jkTargetSlotId = "SlotId"; +static const char* jkTargetExpr = "Expr"; + +static int32_t targetNodeToJson(const void* pObj, SJson* pJson) { + const STargetNode* pNode = (const STargetNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTargetTupleId, pNode->tupleId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTargetSlotId, pNode->slotId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTargetExpr, nodeToJson, pNode->pExpr); + } + + return code; +} + +static const char* jkSlotDescSlotId = "SlotId"; +static const char* jkSlotDescDataType = "DataType"; + +static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { + const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSlotDescSlotId, pNode->slotId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSlotDescDataType, dataTypeToJson, &pNode->dataType); + } + + return code; +} + +static const char* jkTupleDescTupleId = "TupleId"; +static const char* jkTupleDescSlots = "Slots"; + +static int32_t tupleDescNodeToJson(const void* pObj, SJson* pJson) { + const STupleDescNode* pNode = (const STupleDescNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTupleDescTupleId, pNode->tupleId); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkTupleDescSlots, nodeToJson, pNode->pSlots); + } + + return code; +} + +static const char* jkSelectStmtDistinct = "Distinct"; +static const char* jkSelectStmtProjections = "Projections"; +static const char* jkSelectStmtFrom = "From"; +static const char* jkSelectStmtWhere = "Where"; +static const char* jkSelectStmtPartitionBy = "PartitionBy"; +static const char* jkSelectStmtWindow = "Window"; +static const char* jkSelectStmtGroupBy = "GroupBy"; +static const char* jkSelectStmtHaving = "Having"; +static const char* jkSelectStmtOrderBy = "OrderBy"; +static const char* jkSelectStmtLimit = "Limit"; +static const char* jkSelectStmtSlimit = "Slimit"; + +static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { + const SSelectStmt* pNode = (const SSelectStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtProjections, nodeToJson, pNode->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtFrom, nodeToJson, pNode->pFromTable); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtWhere, nodeToJson, pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtPartitionBy, nodeToJson, pNode->pPartitionByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtWindow, nodeToJson, pNode->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtGroupBy, nodeToJson, pNode->pGroupByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtHaving, nodeToJson, pNode->pHaving); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtOrderBy, nodeToJson, pNode->pOrderByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtLimit, nodeToJson, pNode->pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtSlimit, nodeToJson, pNode->pSlimit); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: + return columnNodeToJson(pObj, pJson); case QUERY_NODE_VALUE: + return valueNodeToJson(pObj, pJson); case QUERY_NODE_OPERATOR: + return operatorNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_CONDITION: + return logicConditionNodeToJson(pObj, pJson); case QUERY_NODE_FUNCTION: + return functionNodeToJson(pObj, pJson); case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: + break; case QUERY_NODE_GROUPING_SET: + return groupingSetNodeToJson(pObj, pJson); case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_LIMIT: case QUERY_NODE_STATE_WINDOW: @@ -203,22 +647,35 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_NODE_LIST: case QUERY_NODE_FILL: case QUERY_NODE_COLUMN_REF: + return columnRefNodeToJson(pObj, pJson); case QUERY_NODE_TARGET: + return targetNodeToJson(pObj, pJson); case QUERY_NODE_RAW_EXPR: + break; + case QUERY_NODE_TUPLE_DESC: + return tupleDescNodeToJson(pObj, pJson); + case QUERY_NODE_SLOT_DESC: + return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: + break; case QUERY_NODE_SELECT_STMT: + return selectStmtTojson(pObj, pJson); case QUERY_NODE_SHOW_STMT: break; case QUERY_NODE_LOGIC_PLAN_SCAN: - return logicScanToJson(pObj, pJson); + return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: - return logicJoinToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_FILTER: - return logicFilterToJson(pObj, pJson); + return logicJoinNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_AGG: - return logicAggToJson(pObj, pJson); + return logicAggNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PROJECT: - return logicProjectToJson(pObj, pJson); + return logicProjectNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return physiTagScanNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return physiTableScanNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return physiProjectNodeToJson(pObj, pJson); default: break; } @@ -238,7 +695,7 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) { return code; } -int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { +int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen) { if (NULL == pNode || NULL == pStr || NULL == pLen) { return TSDB_CODE_SUCCESS; } @@ -255,7 +712,7 @@ int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { return code; } - *pStr = tjsonToString(pJson); + *pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson); tjsonDelete(pJson); *pLen = strlen(*pStr) + 1; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 868af93c92..5117fe6f54 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -17,6 +17,7 @@ #include "plannodes.h" #include "taos.h" #include "taoserror.h" +#include "taos.h" #include "thash.h" static SNode* makeNode(ENodeType type, size_t size) { @@ -62,6 +63,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SNodeListNode)); case QUERY_NODE_FILL: return makeNode(type, sizeof(SFillNode)); + case QUERY_NODE_COLUMN_REF: + return makeNode(type, sizeof(SColumnRefNode)); case QUERY_NODE_RAW_EXPR: return makeNode(type, sizeof(SRawExprNode)); case QUERY_NODE_SET_OPERATOR: @@ -74,12 +77,22 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SScanLogicNode)); case QUERY_NODE_LOGIC_PLAN_JOIN: return makeNode(type, sizeof(SJoinLogicNode)); - case QUERY_NODE_LOGIC_PLAN_FILTER: - return makeNode(type, sizeof(SFilterLogicNode)); case QUERY_NODE_LOGIC_PLAN_AGG: return makeNode(type, sizeof(SAggLogicNode)); case QUERY_NODE_LOGIC_PLAN_PROJECT: return makeNode(type, sizeof(SProjectLogicNode)); + case QUERY_NODE_TARGET: + return makeNode(type, sizeof(STargetNode)); + case QUERY_NODE_TUPLE_DESC: + return makeNode(type, sizeof(STupleDescNode)); + case QUERY_NODE_SLOT_DESC: + return makeNode(type, sizeof(SSlotDescNode)); + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return makeNode(type, sizeof(STagScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return makeNode(type, sizeof(STableScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return makeNode(type, sizeof(SProjectPhysiNode)); default: break; } @@ -132,11 +145,22 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { } int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { - pTarget->pTail->pNext = pSrc->pHead; - pSrc->pHead->pPrev = pTarget->pTail; + if (NULL == pTarget || NULL == pSrc) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == pTarget->pHead) { + pTarget->pHead = pSrc->pHead; + } else { + pTarget->pTail->pNext = pSrc->pHead; + if (NULL != pSrc->pHead) { + pSrc->pHead->pPrev = pTarget->pTail; + } + } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; tfree(pSrc); + return TSDB_CODE_SUCCESS; } @@ -172,6 +196,36 @@ void nodesDestroyList(SNodeList* pList) { tfree(pList); } +void* nodesGetValueFromNode(SValueNode *pNode) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + return (void*)&pNode->datum.b; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return (void*)&pNode->datum.i; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + return (void*)&pNode->datum.u; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + return (void*)&pNode->datum.d; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + return (void*)pNode->datum.p; + default: + break; + } + + return NULL; +} + bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); @@ -290,6 +344,10 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* nodesDestroyList(cxt.pCols); return cxt.errCode; } + if (0 == LIST_LENGTH(cxt.pCols)) { + nodesDestroyList(cxt.pCols); + cxt.pCols = NULL; + } *pCols = cxt.pCols; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index d353c245cb..ef040fdff4 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -349,7 +349,7 @@ static SNodeList* getProjectList(SNode* pNode) { return NULL; } -static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, SColumnNode* pCol) { +static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, bool isTag, SColumnNode* pCol) { strcpy(pCol->dbName, pTable->table.dbName); strcpy(pCol->tableAlias, pTable->table.tableAlias); strcpy(pCol->tableName, pTable->table.tableName); @@ -359,7 +359,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p } pCol->tableId = pTable->pMeta->uid; pCol->colId = pColSchema->colId; - // pCol->colType = pColSchema->type; + pCol->colType = isTag ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN; pCol->node.resType.type = pColSchema->type; pCol->node.resType.bytes = pColSchema->bytes; } @@ -383,7 +383,7 @@ static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode if (NULL == pCol) { return generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i < pMeta->tableInfo.numOfTags), pCol); nodesListAppend(pList, (SNode*)pCol); } } else { @@ -408,7 +408,7 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i < pMeta->tableInfo.numOfTags), pCol); found = true; break; } @@ -500,22 +500,24 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { } static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { + varDataSetLen(dst, len); + char* dstVal = varDataVal(dst); // delete escape character: \\, \', \" char delim = src[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { - dst[j] = src[k + 1]; + dstVal[j] = src[k + 1]; cnt++; j++; k++; continue; } - dst[j] = src[k]; + dstVal[j] = src[k]; j++; } - dst[j] = '\0'; + dstVal[j] = '\0'; return j; } @@ -560,7 +562,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { int32_t n = strlen(pVal->literal); - pVal->datum.p = calloc(1, n); + pVal->datum.p = calloc(1, n + VARSTR_HEADER_SIZE); if (NULL == pVal->datum.p) { generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); return DEAL_RES_ERROR; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 4fd0de3803..ec68980c44 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -884,7 +884,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil } assert(src->filterstr == 0 || src->filterstr == 1); - assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0)); return pFilter; } @@ -1507,45 +1507,45 @@ int32_t getTagFilterSerializeLen(SQueryStmtInfo* pQueryInfo) { uint32_t convertRelationalOperator(SToken *pToken) { switch (pToken->type) { case TK_LT: - return TSDB_RELATION_LESS; + return OP_TYPE_LOWER_THAN; case TK_LE: - return TSDB_RELATION_LESS_EQUAL; + return OP_TYPE_LOWER_EQUAL; case TK_GT: - return TSDB_RELATION_GREATER; + return OP_TYPE_GREATER_THAN; case TK_GE: - return TSDB_RELATION_GREATER_EQUAL; + return OP_TYPE_GREATER_EQUAL; case TK_NE: - return TSDB_RELATION_NOT_EQUAL; + return OP_TYPE_NOT_EQUAL; case TK_AND: - return TSDB_RELATION_AND; + return LOGIC_COND_TYPE_AND; case TK_OR: - return TSDB_RELATION_OR; + return LOGIC_COND_TYPE_OR; case TK_EQ: - return TSDB_RELATION_EQUAL; + return OP_TYPE_EQUAL; case TK_PLUS: - return TSDB_BINARY_OP_ADD; + return OP_TYPE_ADD; case TK_MINUS: - return TSDB_BINARY_OP_SUBTRACT; + return OP_TYPE_SUB; case TK_STAR: - return TSDB_BINARY_OP_MULTIPLY; + return OP_TYPE_MULTI; case TK_SLASH: case TK_DIVIDE: - return TSDB_BINARY_OP_DIVIDE; + return OP_TYPE_DIV; case TK_REM: - return TSDB_BINARY_OP_REMAINDER; + return OP_TYPE_MOD; case TK_LIKE: - return TSDB_RELATION_LIKE; + return OP_TYPE_LIKE; case TK_MATCH: - return TSDB_RELATION_MATCH; + return OP_TYPE_MATCH; case TK_NMATCH: - return TSDB_RELATION_NMATCH; + return OP_TYPE_NMATCH; case TK_ISNULL: - return TSDB_RELATION_ISNULL; + return OP_TYPE_IS_NULL; case TK_NOTNULL: - return TSDB_RELATION_NOTNULL; + return OP_TYPE_IS_NOT_NULL; case TK_IN: - return TSDB_RELATION_IN; + return OP_TYPE_IN; default: { return 0; } } } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index d68b04a384..8ed67d7f2f 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -29,9 +29,9 @@ namespace { void generateTestT1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 4) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20).addColumn("c3", TSDB_DATA_TYPE_BIGINT); builder.done(); } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index b408e15abe..b971760132 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -593,7 +593,7 @@ TEST(testCase, function_Test6) { SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pQueryInfo->exprList[1], 0); ASSERT_EQ(p2->pExpr->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(p2->pExpr->_node.optr, TSDB_BINARY_OP_ADD); + ASSERT_EQ(p2->pExpr->_node.optr, OP_TYPE_ADD); ASSERT_EQ(p2->pExpr->_node.pLeft->nodeType, TEXPR_COL_NODE); ASSERT_EQ(p2->pExpr->_node.pRight->nodeType, TEXPR_COL_NODE); diff --git a/source/libs/planner/inc/plannerImpl.h b/source/libs/planner/inc/plannerImpl.h index d89cc26700..559d614829 100644 --- a/source/libs/planner/inc/plannerImpl.h +++ b/source/libs/planner/inc/plannerImpl.h @@ -24,6 +24,7 @@ extern "C" { #include "planner.h" int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode); +int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode); #ifdef __cplusplus } diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index 40a82f4175..be570f0b96 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -18,27 +18,24 @@ #define CHECK_ALLOC(p, res) \ do { \ - if (NULL == p) { \ - printf("%s : %d\n", __FUNCTION__, __LINE__); \ + if (NULL == (p)) { \ pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ - return res; \ + return (res); \ } \ } while (0) #define CHECK_CODE(exec, res) \ do { \ - int32_t code = exec; \ + int32_t code = (exec); \ if (TSDB_CODE_SUCCESS != code) { \ - printf("%s : %d\n", __FUNCTION__, __LINE__); \ pCxt->errCode = code; \ - return res; \ + return (res); \ } \ } while (0) typedef struct SPlanContext { int32_t errCode; int32_t planNodeId; - SNodeList* pResource; } SPlanContext; static SLogicNode* createQueryLogicNode(SPlanContext* pCxt, SNode* pStmt); @@ -60,10 +57,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { FOREACH(pExpr, pCxt->pExprs) { if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - return DEAL_RES_ERROR; - } + CHECK_ALLOC(pCol, DEAL_RES_ERROR); SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); @@ -83,14 +77,30 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - SNode* pNode; - FOREACH(pNode, pExprs) { - if (QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_VALUE == nodeType(pNode)) { - continue; +typedef struct SNameExprCxt { + int32_t planNodeId; + int32_t rewriteId; +} SNameExprCxt; + +static EDealRes doNameExpr(SNode* pNode, void* pContext) { + switch (nodeType(pNode)) { + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { + SNameExprCxt* pCxt = (SNameExprCxt*)pContext; + sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++); + return DEAL_RES_IGNORE_CHILD; } - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", planNodeId, rewriteId); + default: + break; } + + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { + SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId }; + nodesWalkList(pExprs, doNameExpr, &nameCxt); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -135,12 +145,16 @@ static SLogicNode* createScanLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, // set columns to scan SNodeList* pCols = NULL; CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan); - pScan->pScanCols = nodesCloneList(pCols); - CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); + if (NULL != pCols) { + pScan->pScanCols = nodesCloneList(pCols); + CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); + } // set output - pScan->node.pTargets = nodesCloneList(pCols); - CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); + if (NULL != pCols) { + pScan->node.pTargets = nodesCloneList(pCols); + CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); + } return (SLogicNode*)pScan; } @@ -173,8 +187,10 @@ static SLogicNode* createJoinLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); // set on conditions - pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); - CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); + if (NULL != pJoinTable->pOnCond) { + pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); + CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); + } // set the output pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); @@ -200,55 +216,48 @@ static SLogicNode* createLogicNodeByTable(SPlanContext* pCxt, SSelectStmt* pSele return NULL; } -static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SLogicNode* pChild, SSelectStmt* pSelect) { - if (NULL == pSelect->pWhere) { - return NULL; +typedef struct SCreateColumnCxt { + int32_t errCode; + SNodeList* pList; +} SCreateColumnCxt; + +static EDealRes doCreateColumn(SNode* pNode, void* pContext) { + SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: { + SNode* pCol = nodesCloneNode(pNode); + CHECK_ALLOC(pCol, DEAL_RES_ERROR); + CHECK_CODE(nodesListAppend(pCxt->pList, pCol), DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { + SExprNode* pExpr = (SExprNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + CHECK_ALLOC(pCol, DEAL_RES_ERROR); + pCol->node.resType = pExpr->resType; + strcpy(pCol->colName, pExpr->aliasName); + CHECK_CODE(nodesListAppend(pCxt->pList, (SNode*)pCol), DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; + } + default: + break; } - SFilterLogicNode* pFilter = (SFilterLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILTER); - CHECK_ALLOC(pFilter, NULL); - pFilter->node.id = pCxt->planNodeId++; - - // set filter conditions - pFilter->node.pConditions = nodesCloneNode(pSelect->pWhere); - CHECK_ALLOC(pFilter->node.pConditions, (SLogicNode*)pFilter); - - // set the output - pFilter->node.pTargets = nodesCloneList(pChild->pTargets); - CHECK_ALLOC(pFilter->node.pTargets, (SLogicNode*)pFilter); - - return (SLogicNode*)pFilter; + return DEAL_RES_CONTINUE; } static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) { - SNodeList* pList = nodesMakeList(); - CHECK_ALLOC(pList, NULL); - SNode* pNode; - FOREACH(pNode, pExprs) { - if (QUERY_NODE_VALUE == nodeType(pNode)) { - continue; - } else if (QUERY_NODE_COLUMN == nodeType(pNode)) { - SNode* pCol = nodesCloneNode(pNode); - if (NULL == pCol) { - goto error; - } - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, pCol)) { - goto error; - } - } else { - SExprNode* pExpr = (SExprNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - goto error; - } - pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); - } + SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; + CHECK_ALLOC(cxt.pList, NULL); + + nodesWalkList(pExprs, doCreateColumn, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pList); + return NULL; } - return pList; -error: - nodesDestroyList(pList); - return NULL; + return cxt.pList; } static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) { @@ -263,24 +272,37 @@ static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) pAgg->node.id = pCxt->planNodeId++; // set grouyp keys, agg funcs and having conditions - pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); - CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); - pAgg->pAggFuncs = nodesCloneList(pAggFuncs); - CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); + if (NULL != pSelect->pGroupByList) { + pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); + CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); + } + if (NULL != pAggFuncs) { + pAgg->pAggFuncs = nodesCloneList(pAggFuncs); + CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); + } // rewrite the expression in subsequent clauses CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); - pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); - CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); + if (NULL != pSelect->pHaving) { + pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); + CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); + } // set the output - pAgg->node.pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); + pAgg->node.pTargets = nodesMakeList(); CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); - SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); - CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); - nodesListAppendList(pAgg->node.pTargets, pTargets); + if (NULL != pAgg->pGroupKeys) { + SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); + CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); + nodesListAppendList(pAgg->node.pTargets, pTargets); + } + if (NULL != pAgg->pAggFuncs) { + SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); + CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); + nodesListAppendList(pAgg->node.pTargets, pTargets); + } return (SLogicNode*)pAgg; } @@ -322,8 +344,9 @@ static SLogicNode* createProjectLogicNode(SPlanContext* pCxt, SSelectStmt* pSele static SLogicNode* createSelectLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) { SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createWhereFilterLogicNode(pCxt, pRoot, pSelect)); + if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) { + pRoot->pConditions = nodesCloneNode(pSelect->pWhere); + CHECK_ALLOC(pRoot->pConditions, pRoot); } if (TSDB_CODE_SUCCESS == pCxt->errCode) { pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect)); @@ -344,7 +367,7 @@ static SLogicNode* createQueryLogicNode(SPlanContext* pCxt, SNode* pStmt) { } int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) { - SPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 0 }; + SPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1 }; SLogicNode* pRoot = createQueryLogicNode(&cxt, pNode); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode((SNode*)pRoot); @@ -353,3 +376,300 @@ int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) { *pLogicNode = pRoot; return TSDB_CODE_SUCCESS; } + +int32_t optimize(SLogicNode* pLogicNode) { + // todo + return TSDB_CODE_SUCCESS; +} + +typedef struct SSubLogicPlan { + SNode* pRoot; // SLogicNode + bool haveSuperTable; + bool haveSystemTable; +} SSubLogicPlan; + +int32_t splitLogicPlan(SSubLogicPlan* pLogicPlan) { + // todo + return TSDB_CODE_SUCCESS; +} + +typedef struct SSlotIndex { + int16_t tupleId; + int16_t slotId; +} SSlotIndex; + +typedef struct SPhysiPlanContext { + int32_t errCode; + int16_t nextTupleId; + SArray* pTupleHelper; +} SPhysiPlanContext; + +static int32_t getSlotKey(SNode* pNode, char* pKey) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName); + } else { + return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); + } +} + +static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) { + SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF); + if (NULL == pCol) { + return NULL; + } + pCol->dataType = ((SExprNode*)pNode)->resType; + pCol->tupleId = tupleId; + pCol->slotId = slotId; + pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1); + return (SNode*)pCol; +} + +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { + SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); + CHECK_ALLOC(pSlot, NULL); + pSlot->slotId = slotId; + pSlot->dataType = ((SExprNode*)pNode)->resType; + pSlot->srcTupleId = -1; + pSlot->srcSlotId = -1; + pSlot->reserve = false; + pSlot->output = true; + return (SNode*)pSlot; +} + +static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) { + STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); + if (NULL == pTarget) { + return NULL; + } + pTarget->tupleId = tupleId; + pTarget->slotId = slotId; + pTarget->pExpr = nodesCloneNode(pNode); + if (NULL == pTarget->pExpr) { + nodesDestroyNode((SNode*)pTarget); + return NULL; + } + return (SNode*)pTarget; +} + +static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) { + pTuple->tupleId = pCxt->nextTupleId++; + + SHashObj* pHash = NULL; + if (NULL == pTuple->pSlots) { + pTuple->pSlots = nodesMakeList(); + CHECK_ALLOC(pTuple->pSlots, TSDB_CODE_OUT_OF_MEMORY); + + pHash = taosHashInit(LIST_LENGTH(pList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayInsert(pCxt->pTupleHelper, pTuple->tupleId, &pHash)) { + taosHashCleanup(pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); + } + + *pOutput = nodesMakeList(); + CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY); + + SNode* pNode = NULL; + int16_t slotId = 0; + FOREACH(pNode, pList) { + SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); + CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); + if (TSDB_CODE_SUCCESS != nodesListAppend(pTuple->pSlots, (SNode*)pSlot)) { + nodesDestroyNode(pSlot); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); + CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); + if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) { + nodesDestroyNode(pTarget); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId }; + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + int32_t len = getSlotKey(pNode, name); + CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); + + ++slotId; + } + return TSDB_CODE_SUCCESS; +} + +typedef struct STransformCxt { + int32_t errCode; + SHashObj* pHash; +} STransformCxt; + +static EDealRes doTransform(SNode** pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + STransformCxt* pCxt = (STransformCxt*)pContext; + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + int32_t len = getSlotKey(*pNode, name); + SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len); + if (NULL != pIndex) { + *pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId); + CHECK_ALLOC(*pNode, DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; + } + } + return DEAL_RES_CONTINUE; +} + +static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) { + SNode* pRes = nodesCloneNode(pNode); + CHECK_ALLOC(pRes, NULL); + STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; + nodesRewriteNode(&pRes, doTransform, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyNode(pRes); + return NULL; + } + return pRes; +} + +static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) { + SNodeList* pRes = nodesCloneList(pList); + CHECK_ALLOC(pRes, NULL); + STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; + nodesRewriteList(pRes, doTransform, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(pRes); + return NULL; + } + return pRes; +} + +static SPhysiNode* makePhysiNode(ENodeType type) { + SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); + if (NULL == pPhysiNode) { + return NULL; + } + pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC; + return pPhysiNode; +} + +static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { + CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); + + if (NULL != pScanLogicNode->node.pConditions) { + pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions); + CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY); + } + + pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; + pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; + pScanPhysiNode->order = TSDB_ORDER_ASC; + pScanPhysiNode->count = 1; + pScanPhysiNode->reverse = 0; + + return TSDB_CODE_SUCCESS; +} + +static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + CHECK_ALLOC(pTagScan, NULL); + CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); + return (SPhysiNode*)pTagScan; +} + +static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + CHECK_ALLOC(pTableScan, NULL); + CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); + pTableScan->scanFlag = pScanLogicNode->scanFlag; + pTableScan->scanRange = pScanLogicNode->scanRange; + return (SPhysiNode*)pTableScan; +} + +static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + switch (pScanLogicNode->scanType) { + case SCAN_TYPE_TAG: + return createTagScanPhysiNode(pCxt, pScanLogicNode); + case SCAN_TYPE_TABLE: + return createTableScanPhysiNode(pCxt, pScanLogicNode); + case SCAN_TYPE_STABLE: + case SCAN_TYPE_STREAM: + break; + default: + break; + } +} + +static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) { + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT); + CHECK_ALLOC(pProject, NULL); + + SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections); + CHECK_ALLOC(pProjections, (SPhysiNode*)pProject); + CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject); + nodesDestroyList(pProjections); + + if (NULL != pProjectLogicNode->node.pConditions) { + pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions); + CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject); + } + + return (SPhysiNode*)pProject; +} + +static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) { + SNodeList* pChildern = nodesMakeList(); + CHECK_ALLOC(pChildern, NULL); + + SNode* pLogicChild; + FOREACH(pLogicChild, pLogicPlan->pChildren) { + SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild); + if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + nodesDestroyList(pChildern); + return NULL; + } + } + + SPhysiNode* pPhyNode = NULL; + switch (nodeType(pLogicPlan)) { + case QUERY_NODE_LOGIC_PLAN_SCAN: + pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan); + break; + case QUERY_NODE_LOGIC_PLAN_JOIN: + break; + case QUERY_NODE_LOGIC_PLAN_AGG: + break; + case QUERY_NODE_LOGIC_PLAN_PROJECT: + pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan); + break; + default: + break; + } + + if (NULL != pPhyNode) { + pPhyNode->pChildren = pChildern; + SNode* pChild; + FOREACH(pChild, pPhyNode->pChildren) { + ((SPhysiNode*)pChild)->pParent = pPhyNode; + } + } + + return pPhyNode; +} + +int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) { + SPhysiPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .nextTupleId = 0, .pTupleHelper = taosArrayInit(32, POINTER_BYTES) }; + if (NULL == cxt.pTupleHelper) { + return TSDB_CODE_OUT_OF_MEMORY; + } + *pPhyNode = createPhysiNode(&cxt, pLogicNode); + return cxt.errCode; +} + +int32_t buildPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) { + // split + // scale out + // maping + // create + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/test/newPlannerTest.cpp b/source/libs/planner/test/newPlannerTest.cpp index c227bf88ba..e99f0c150c 100644 --- a/source/libs/planner/test/newPlannerTest.cpp +++ b/source/libs/planner/test/newPlannerTest.cpp @@ -25,6 +25,11 @@ using namespace testing; class NewPlannerTest : public Test { protected: + enum TestTarget { + TEST_LOGIC_PLAN, + TEST_PHYSICAL_PLAN + }; + void setDatabase(const string& acctId, const string& db) { acctId_ = acctId; db_ = db; @@ -40,28 +45,40 @@ protected: cxt_.pSql = sqlBuf_.c_str(); } - bool run() { + bool run(TestTarget target = TEST_PHYSICAL_PLAN) { int32_t code = parser(&cxt_, &query_); - // cout << "parser return " << code << endl; + if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] parser code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + cout << "sql:[" << cxt_.pSql << "] parser code:" << code << ", strerror:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; return false; } + + const string syntaxTreeStr = toString(query_.pRoot, false); + SLogicNode* pLogicPlan = nullptr; code = createLogicPlan(query_.pRoot, &pLogicPlan); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] plan code:" << tstrerror(code) << endl; + cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; return false; } - char* pStr = NULL; - int32_t len = 0; - code = nodesNodeToString((const SNode*)pLogicPlan, &pStr, &len); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] toString code:" << tstrerror(code) << endl; - return false; + + cout << "sql : [" << cxt_.pSql << "]" << endl; + cout << "syntax test : " << endl; + cout << syntaxTreeStr << endl; + cout << "unformatted logic plan : " << endl; + cout << toString((const SNode*)pLogicPlan, false) << endl; + + if (TEST_PHYSICAL_PLAN == target) { + SPhysiNode* pPhyPlan = nullptr; + code = createPhysiPlan(pLogicPlan, &pPhyPlan); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] physical plan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + cout << "unformatted physical plan : " << endl; + cout << toString((const SNode*)pPhyPlan, false) << endl; } - cout << "logic plan : " << endl; - cout << pStr << endl; + return true; } @@ -75,6 +92,19 @@ private: cxt_.msgLen = max_err_len; } + string toString(const SNode* pRoot, bool format = true) { + char* pStr = NULL; + int32_t len = 0; + int32_t code = nodesNodeToString(pRoot, format, &pStr, &len); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; + return string(); + } + string str(pStr); + tfree(pStr); + return str; + } + string acctId_; string db_; char errMagBuf_[max_err_len]; @@ -89,3 +119,26 @@ TEST_F(NewPlannerTest, simple) { bind("SELECT * FROM t1"); ASSERT_TRUE(run()); } + +TEST_F(NewPlannerTest, groupBy) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); + ASSERT_TRUE(run()); + + bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); + ASSERT_TRUE(run()); + + bind("SELECT c1 + c3, count(*) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); + ASSERT_TRUE(run()); +} + +TEST_F(NewPlannerTest, subquery) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM (SELECT c1 + c3 a, c1 + count(*) b FROM t1 where c2 = 'abc' GROUP BY c1, c3) where a > 100 group by b"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt new file mode 100644 index 0000000000..d2d02bc0dc --- /dev/null +++ b/source/libs/scalar/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src SCALAR_SRC) + +add_library(scalar STATIC ${SCALAR_SRC}) +target_include_directories( + scalar + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(scalar + PRIVATE os util common nodes function qcom + ) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/inc/tfilter.h b/source/libs/scalar/inc/filterInt.h similarity index 69% rename from source/libs/executor/inc/tfilter.h rename to source/libs/scalar/inc/filterInt.h index 55edf27949..4700fc6d6e 100644 --- a/source/libs/executor/inc/tfilter.h +++ b/source/libs/scalar/inc/filterInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_QFILTER_H -#define TDENGINE_QFILTER_H +#ifndef TDENGINE_FILTER_INT_H +#define TDENGINE_FILTER_INT_H #ifdef __cplusplus extern "C" { @@ -22,7 +22,10 @@ extern "C" { #include "thash.h" #include "tname.h" -#include "function.h" +#include "common.h" +#include "scalar.h" +#include "querynodes.h" +#include "query.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -40,7 +43,6 @@ enum { FLD_TYPE_COLUMN = 1, FLD_TYPE_VALUE = 2, FLD_TYPE_MAX = 3, - FLD_DESC_NO_FREE = 4, FLD_DATA_NO_FREE = 8, FLD_DATA_IS_HASH = 16, }; @@ -58,11 +60,6 @@ enum { RANGE_FLG_NULL = 4, }; -enum { - FI_OPTION_NO_REWRITE = 1, - FI_OPTION_TIMESTAMP = 2, - FI_OPTION_NEED_UNIQE = 4, -}; enum { FI_STATUS_ALL = 1, @@ -83,6 +80,12 @@ enum { RANGE_TYPE_MR_CTX = 3, }; +enum { + FI_ACTION_NO_NEED = 1, + FI_ACTION_CONTINUE, + FI_ACTION_STOP, +}; + typedef struct OptrStr { uint16_t optr; char *str; @@ -95,17 +98,10 @@ typedef struct SFilterRange { char eflag; } SFilterRange; -typedef struct SFilterColRange { - uint16_t idx; //column field idx - bool isNull; - bool notNull; - bool isRange; - SFilterRange ra; -} SFilterColRange; - typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); +typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { int64_t s; @@ -153,37 +149,39 @@ typedef struct SFilterField { } SFilterField; typedef struct SFilterFields { - uint16_t size; - uint16_t num; + uint32_t size; + uint32_t num; SFilterField *fields; } SFilterFields; typedef struct SFilterFieldId { uint16_t type; - uint16_t idx; + uint32_t idx; } SFilterFieldId; typedef struct SFilterGroup { - uint16_t unitSize; - uint16_t unitNum; - uint16_t *unitIdxs; + uint32_t unitSize; + uint32_t unitNum; + uint32_t *unitIdxs; uint8_t *unitFlags; // !unit result } SFilterGroup; typedef struct SFilterColInfo { uint8_t type; int32_t dataType; + uint8_t optr; // for equal operation in the relation of RELATION_IN + int64_t value; // for equal operation in the relation of RELATION_IN void *info; } SFilterColInfo; typedef struct SFilterGroupCtx { - uint16_t colNum; - uint16_t *colIdx; + uint32_t colNum; + uint32_t *colIdx; SFilterColInfo *colInfo; } SFilterGroupCtx; typedef struct SFilterColCtx { - uint16_t colIdx; + uint32_t colIdx; void* ctx; } SFilterColCtx; @@ -217,16 +215,32 @@ typedef struct SFilterPCtx { SHashObj *unitHash; } SFilterPCtx; +typedef struct SFltTreeStat { + int32_t code; + bool scalarMode; +} SFltTreeStat; + +typedef struct SFltScalarCtx { + SNode *node; +} SFltScalarCtx; + +typedef struct SFltBuildGroupCtx { + SFilterInfo *info; + SArray *group; + int32_t code; +} SFltBuildGroupCtx; + typedef struct SFilterInfo { + bool scalarMode; + SFltScalarCtx sclCtx; uint32_t options; uint32_t status; - uint16_t unitSize; - uint16_t unitNum; - uint16_t groupNum; - uint16_t colRangeNum; + uint32_t unitSize; + uint32_t unitNum; + uint32_t groupNum; + uint32_t colRangeNum; SFilterFields fields[FLD_TYPE_MAX]; SFilterGroup *groups; - uint16_t *cgroups; SFilterUnit *units; SFilterComUnit *cunits; uint8_t *unitRes; // result @@ -234,22 +248,21 @@ typedef struct SFilterInfo { SFilterRangeCtx **colRange; filter_exec_func func; uint8_t blkFlag; - uint16_t blkGroupNum; - uint16_t *blkUnits; + uint32_t blkGroupNum; + uint32_t *blkUnits; int8_t *blkUnitRes; - + void *pTable; + SFilterPCtx pctx; } SFilterInfo; -#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) - -#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) -#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) +#define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) -#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) -#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) +#define SET_AND_OPTR(ctx, o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) +#define SET_OR_OPTR(ctx,o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) #define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) #define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) @@ -259,7 +272,7 @@ typedef struct SFilterInfo { #define FILTER_CLR_FLAG(st, f) st &= (~f) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) -#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) +#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0) #define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) #define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) @@ -269,25 +282,33 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) -#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) -#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) -#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) +#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL) + +#define fltFatal(...) qFatal(__VA_ARGS__) +#define fltError(...) qError(__VA_ARGS__) +#define fltWarn(...) qWarn(__VA_ARGS__) +#define fltInfo(...) qInfo(__VA_ARGS__) +#define fltDebug(...) qDebug(__VA_ARGS__) +#define fltTrace(...) qTrace(__VA_ARGS__) + + +#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0) +#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define CHK_RETV(c) do { if (c) { return; } } while (0) -#define CHK_RET(c, r) do { if (c) { return r; } } while (0) -#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) -#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) -#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) -#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) +#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) +#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -300,6 +321,7 @@ typedef struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) +#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) @@ -314,7 +336,7 @@ typedef struct SFilterInfo { #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) @@ -322,18 +344,12 @@ typedef struct SFilterInfo { #define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) +extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); +extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); -extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); -extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); -extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); -extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); -extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus } #endif -#endif // TDENGINE_QFILTER_H +#endif // TDENGINE_FILTER_INT_H diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h new file mode 100644 index 0000000000..41ee90667e --- /dev/null +++ b/source/libs/scalar/inc/sclInt.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_SCALARINT_H +#define TDENGINE_SCALARINT_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "common.h" +#include "thash.h" +#include "query.h" + +typedef struct SScalarCtx { + int32_t code; + SSDataBlock *pSrc; + SHashObj *pRes; /* element is SScalarParam */ +} SScalarCtx; + + +#define SCL_DATA_TYPE_DUMMY_HASH 9000 +#define SCL_DEFAULT_OP_NUM 10 + +#define sclFatal(...) qFatal(__VA_ARGS__) +#define sclError(...) qError(__VA_ARGS__) +#define sclWarn(...) qWarn(__VA_ARGS__) +#define sclInfo(...) qInfo(__VA_ARGS__) +#define sclDebug(...) qDebug(__VA_ARGS__) +#define sclTrace(...) qTrace(__VA_ARGS__) + +#define SCL_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SCALARINT_H \ No newline at end of file diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/scalar/inc/sclfunc.h similarity index 89% rename from source/libs/function/inc/tscalarfunction.h rename to source/libs/scalar/inc/sclfunc.h index 6d23775610..8e03470033 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/scalar/inc/sclfunc.h @@ -20,13 +20,7 @@ extern "C" { #endif #include "function.h" - -typedef struct SScalarFuncParam { - void* data; - int32_t num; - int32_t type; - int32_t bytes; -} SScalarFuncParam; +#include "scalar.h" typedef struct SScalarFunctionSupport { struct SExprInfo *pExprInfo; @@ -39,7 +33,7 @@ typedef struct SScalarFunctionSupport { extern struct SScalarFunctionInfo scalarFunc[8]; -int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, +int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); diff --git a/source/libs/function/inc/tbinoperator.h b/source/libs/scalar/inc/sclvector.h similarity index 83% rename from source/libs/function/inc/tbinoperator.h rename to source/libs/scalar/inc/sclvector.h index c1b6b0bc31..55c4828745 100644 --- a/source/libs/function/inc/tbinoperator.h +++ b/source/libs/scalar/inc/sclvector.h @@ -20,11 +20,10 @@ extern "C" { #endif -#include "tscalarfunction.h" +#include "sclfunc.h" -typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order); +typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); -bool isBinaryStringOp(int32_t op); #ifdef __cplusplus } diff --git a/source/libs/executor/src/tfilter.c b/source/libs/scalar/src/filter.c similarity index 66% rename from source/libs/executor/src/tfilter.c rename to source/libs/scalar/src/filter.c index 97dccb5c7b..95b3ea922a 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/scalar/src/filter.c @@ -17,59 +17,67 @@ #include "thash.h" //#include "queryLog.h" #include "tcompare.h" -#include "tfilter.h" +#include "filterInt.h" +#include "filter.h" OptrStr gOptrStr[] = { - {TSDB_RELATION_INVALID, "invalid"}, - {TSDB_RELATION_LESS, "<"}, - {TSDB_RELATION_GREATER, ">"}, - {TSDB_RELATION_EQUAL, "="}, - {TSDB_RELATION_LESS_EQUAL, "<="}, - {TSDB_RELATION_GREATER_EQUAL, ">="}, - {TSDB_RELATION_NOT_EQUAL, "!="}, - {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_MATCH, "nmatch"}, - {TSDB_RELATION_ISNULL, "is null"}, - {TSDB_RELATION_NOTNULL, "not null"}, - {TSDB_RELATION_IN, "in"}, - {TSDB_RELATION_AND, "and"}, - {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"} -}; + {0, "invalid"}, + {OP_TYPE_ADD, "+"}, + {OP_TYPE_SUB, "-"}, + {OP_TYPE_MULTI, "*"}, + {OP_TYPE_DIV, "/"}, + {OP_TYPE_MOD, "%"}, -static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { - const SSchema *sch1 = desc1; - const SSchema *sch2 = desc2; + // bit operator + {OP_TYPE_BIT_AND, "&"}, + {OP_TYPE_BIT_OR, "|"}, - return sch1->colId != sch2->colId; -} + // comparison operator + {OP_TYPE_GREATER_THAN, ">"}, + {OP_TYPE_GREATER_EQUAL, ">="}, + {OP_TYPE_LOWER_THAN, "<"}, + {OP_TYPE_LOWER_EQUAL, "<="}, + {OP_TYPE_EQUAL, "=="}, + {OP_TYPE_NOT_EQUAL, "!="}, + {OP_TYPE_IN, "in"}, + {OP_TYPE_NOT_IN, "not in"}, + {OP_TYPE_LIKE, "like"}, + {OP_TYPE_NOT_LIKE, "not like"}, + {OP_TYPE_MATCH, "match"}, + {OP_TYPE_NMATCH, "nmatch"}, + {OP_TYPE_IS_NULL, "is null"}, + {OP_TYPE_IS_NOT_NULL, "not null"}, + {OP_TYPE_IS_TRUE, "is true"}, + {OP_TYPE_IS_FALSE, "is false"}, + {OP_TYPE_IS_UNKNOWN, "is unknown"}, + {OP_TYPE_IS_NOT_TRUE, "not true"}, + {OP_TYPE_IS_NOT_FALSE, "not false"}, + {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, -static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { - const SVariant *val1 = desc1; - const SVariant *val2 = desc2; - - return taosVariantCompare(val1, val2); -} - - -filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { - NULL, - filterFieldColDescCompare, - filterFieldValDescCompare + // json operator + {OP_TYPE_JSON_GET_VALUE, "json get"}, + {OP_TYPE_JSON_CONTAINS, "json contains"} }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0; + int32_t result = cfunc(maxv, minr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0; + int32_t result = cfunc(maxv, minr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) <= 0; + int32_t result = cfunc(minv, maxr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) < 0; + int32_t result = cfunc(minv, maxr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; @@ -122,30 +130,30 @@ rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRan int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { if (optr2) { - assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); + assert(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); - if (optr == TSDB_RELATION_GREATER) { - if (optr2 == TSDB_RELATION_LESS) { + if (optr == OP_TYPE_GREATER_THAN) { + if (optr2 == OP_TYPE_LOWER_THAN) { return 0; } return 1; } - if (optr2 == TSDB_RELATION_LESS) { + if (optr2 == OP_TYPE_LOWER_THAN) { return 2; } return 3; } else { switch (optr) { - case TSDB_RELATION_GREATER: + case OP_TYPE_GREATER_THAN: return 4; - case TSDB_RELATION_GREATER_EQUAL: + case OP_TYPE_GREATER_EQUAL: return 5; - case TSDB_RELATION_LESS: + case OP_TYPE_LOWER_THAN: return 6; - case TSDB_RELATION_LESS_EQUAL: + case OP_TYPE_LOWER_EQUAL: return 7; default: break; @@ -156,15 +164,17 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { } __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, - compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, + compareDoubleVal, compareLenPrefixedStr, compareStrPatternMatch, compareChkInString, compareWStrPatternMatch, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, - setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch + setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, + compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t comparFn = 0; - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -186,6 +196,29 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { assert(0); } } + + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return 21; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return 22; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return 23; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return 24; + default: + assert(0); + } + } switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -197,14 +230,18 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = 26; + } else if (optr == OP_TYPE_IN) { comparFn = 8; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = 25; } else { /* normal relational comparFn */ comparFn = 6; } @@ -213,14 +250,18 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = 9; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_LIKE) { + comparFn = 27; + } else if (optr == OP_TYPE_IN) { comparFn = 8; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = 25; } else { comparFn = 10; } @@ -240,6 +281,10 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { return comparFn; } +__compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) { + return gDataCompare[filterGetCompFuncIdx(type, optr)]; +} + static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); @@ -254,7 +299,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) { info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); + info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); @@ -281,7 +326,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt void* filterInitRangeCtx(int32_t type, int32_t options) { if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - //qError("not supported range type:%d", type); + qError("not supported range type:%d", type); return NULL; } @@ -358,7 +403,7 @@ int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { SET_AND_OPTR(ctx, raOptr); if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); @@ -382,8 +427,8 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { if (ctx->rs == NULL) { if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) - || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) - || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { + || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == LOGIC_COND_TYPE_AND)) + || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == LOGIC_COND_TYPE_OR))) { APPEND_RANGE(ctx, ctx->rs, ra); FILTER_SET_FLAG(ctx->status, MR_ST_START); } @@ -395,7 +440,7 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { SFilterRangeNode *rn = NULL; int32_t cr = 0; - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { while (r != NULL) { cr = ctx->pCompareFunc(&r->ra.s, &ra->e); if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { @@ -507,7 +552,7 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { if (notnull) { bool all = false; FREE_FROM_RANGE(ctx, ctx->rs); - filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); + filterAddRangeOptr(h, OP_TYPE_IS_NOT_NULL, optr, NULL, &all); if (all) { FILTER_SET_FLAG(ctx->status, MR_ST_ALL); } @@ -538,7 +583,7 @@ int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - assert(optr == TSDB_RELATION_OR); + assert(optr == LOGIC_COND_TYPE_OR); if (sctx->rs == NULL) { return TSDB_CODE_SUCCESS; @@ -589,13 +634,13 @@ int32_t filterFinishRange(void* h) { return TSDB_CODE_SUCCESS; } - if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { + if (FILTER_GET_FLAG(ctx->options, FLT_OPTION_TIMESTAMP)) { SFilterRangeNode *r = ctx->rs; SFilterRangeNode *rn = NULL; while (r && r->next) { int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); + operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { rn = r->next; SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); @@ -648,7 +693,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) { } if (num == 0) { - //qError("no range result"); + qError("no range result"); return TSDB_CODE_QRY_APP_ERROR; } @@ -660,14 +705,14 @@ int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; if (src->isnull){ - filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); + filterAddRangeOptr(ctx, OP_TYPE_IS_NULL, optr, empty, all); if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { *all = true; } } if (src->notnull) { - filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); + filterAddRangeOptr(ctx, OP_TYPE_IS_NOT_NULL, optr, empty, all); if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { *all = true; } @@ -676,7 +721,7 @@ int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, if (src->isrange) { filterAddRangeOptr(ctx, 0, optr, empty, all); - if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { + if (!(optr == LOGIC_COND_TYPE_OR && ctx->notnull)) { filterAddRangeCtx(ctx, src, optr); } @@ -738,8 +783,28 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { int32_t leftSize = (int32_t)taosArrayGetSize(left); int32_t rightSize = (int32_t)taosArrayGetSize(right); -// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); -// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + if (taosArrayGetSize(left) <= 0) { + if (taosArrayGetSize(right) <= 0) { + fltError("both groups are empty"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SFilterGroup *gp = NULL; + while (gp = (SFilterGroup *)taosArrayPop(right)) { + taosArrayPush(group, gp); + } + + return TSDB_CODE_SUCCESS; + } + + if (taosArrayGetSize(right) <= 0) { + SFilterGroup *gp = NULL; + while (gp = (SFilterGroup *)taosArrayPop(left)) { + taosArrayPush(group, gp); + } + + return TSDB_CODE_SUCCESS; + } for (int32_t l = 0; l < leftSize; ++l) { SFilterGroup *gp1 = taosArrayGet(left, l); @@ -756,8 +821,8 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { } int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { - for (uint16_t i = 0; i < fields->num; ++i) { - if (0 == gDescCompare[type](fields->fields[i].desc, v)) { + for (uint32_t i = 0; i < fields->num; ++i) { + if (nodesEqualNode(fields->fields[i].desc, v)) { return i; } } @@ -769,7 +834,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { if (type == FLD_TYPE_VALUE) { if (info->pctx.valHash == false) { - //qError("value hash is empty"); + qError("value hash is empty"); return -1; } @@ -782,17 +847,18 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d return -1; } - +// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0 +// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t idx = -1; - uint16_t *num; + uint32_t *num; num = &info->fields[type].num; if (*num > 0) { if (type == FLD_TYPE_COLUMN) { idx = filterGetFiledByDesc(&info->fields[type], type, desc); - } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { idx = filterGetFiledByData(info, type, *data, dataLen); } } @@ -814,7 +880,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, ++(*num); - if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.valHash == NULL) { info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); } @@ -822,10 +888,6 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); } } else { - if (freeIfExists) { - tfree(desc); - } - if (data && freeIfExists) { tfree(*data); } @@ -840,28 +902,31 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); - FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); return TSDB_CODE_SUCCESS; } -int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { -// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); -// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR); +int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { + if (node == NULL) { + fltError("empty node"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) { + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } int32_t type; void *v; - if (node->nodeType == TEXPR_BINARYEXPR_NODE) { + if (nodeType(node) == QUERY_NODE_COLUMN_REF) { type = FLD_TYPE_COLUMN; - v = node->pSchema; - node->pSchema = NULL; + v = node; } else { type = FLD_TYPE_VALUE; - v = node->pVal; - node->pVal = NULL; + v = node; } filterAddField(info, v, NULL, type, fid, 0, true); @@ -869,8 +934,8 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI return TSDB_CODE_SUCCESS; } -int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { + if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.unitHash == NULL) { info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); } else { @@ -878,14 +943,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); if (hu) { - *uidx = *(uint16_t *)hu; + *uidx = *(uint32_t *)hu; return TSDB_CODE_SUCCESS; } } } if (info->unitNum >= info->unitSize) { - uint16_t psize = info->unitSize; + uint32_t psize = info->unitSize; info->unitSize += FILTER_DEFAULT_UNIT_SIZE; info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); @@ -903,7 +968,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { - assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); + if(optr != OP_TYPE_IS_NULL && optr != OP_TYPE_IS_NOT_NULL && optr != FILTER_DUMMY_EMPTY_OPTR){ + return -1; + } } SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); @@ -913,7 +980,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi *uidx = info->unitNum; - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { int64_t v = 0; FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); @@ -926,7 +993,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi -int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { +int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { group->unitSize += FILTER_DEFAULT_UNIT_SIZE; group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); @@ -937,275 +1004,53 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { return TSDB_CODE_SUCCESS; } -int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { - SBufferReader br = tbufInitReader(buf, len, false); - uint32_t sType = tbufReadUint32(&br); - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); +int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { + SOperatorNode *node = (SOperatorNode *)tree; + int32_t ret = TSDB_CODE_SUCCESS; + SFilterFieldId left = {0}, right = {0}; + filterAddFieldFromNode(info, node->pLeft, &left); + uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + int32_t len = 0; + uint32_t uidx = 0; int32_t code = 0; - int dummy = -1; - SVariant tmpVar = {0}; - size_t t = 0; - int32_t sz = tbufReadInt32(&br); - void *pvar = NULL; - int64_t val = 0; - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(sType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = 128; - } - - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { + SNodeListNode *listNode = (SNodeListNode *)node->pRight; + void *fdata = NULL; + SListCell *cell = listNode->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; - for (int32_t i = 0; i < sz; i++) { - switch (sType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)&val = tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(float *)&val = (float)tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_BINARY: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - default: - taosHashCleanup(pObj); - *q = NULL; - assert(0); - } - - taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); + for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + out.data = malloc(sizeof(int64_t)); - if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); - bufLen = (int32_t)t; - } - - bool converted = false; - char extInfo = 0; - - switch (tType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_FLOAT: { -// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } - -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_BINARY: { - if (taosVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - if (taosVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - default: - goto _return; - } - - taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); - taosVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - } - - *q = (void *)pObj; - pObj = NULL; - -_return: - taosVariantDestroy(&tmpVar); - taosHashCleanup(pObj); - tfree(tmp); - - return code; -} - - - -int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { - SFilterFieldId left = {0}, right = {0}; - - filterAddFieldFromNode(info, tree->_node.pLeft, &left); - - SVariant* var = tree->_node.pRight->pVal; - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); - size_t len = 0; - uint16_t uidx = 0; - - if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { - void *data = NULL; - filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); -// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); - - if (taosHashGetSize((SHashObj *)data) <= 0) { - filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - taosHashCleanup(data); - - return TSDB_CODE_SUCCESS; - } - - void *p = taosHashIterate((SHashObj *)data, NULL); - while(p) { - void* key = NULL; - len = 0; - - taosHashGetKey(p, &key, &len); - void *fdata = NULL; - - if (IS_VAR_DATA_TYPE(type)) { - fdata = malloc(len + VARSTR_HEADER_SIZE); - varDataLen(fdata) = len; - memcpy(varDataVal(fdata), key, len); - len += VARSTR_HEADER_SIZE; - } else { - fdata = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(fdata, key); - len = tDataTypes[type].bytes; + code = vectorConvertImpl(&in, &out); + if (code) { + fltError("convert from %d to %d failed", in.type, out.type); + tfree(out.data); + FLT_ERR_RET(code); } + len = tDataTypes[type].bytes; + filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); - filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); taosArrayPush(group, &fgroup); - - p = taosHashIterate((SHashObj *)data, p); - } - taosHashCleanup(data); + cell = cell->pNext; + } } else { - filterAddFieldFromNode(info, tree->_node.pRight, &right); + filterAddFieldFromNode(info, node->pRight, &right); - filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + FLT_ERR_RET(filterAddUnit(info, node->opType, &left, &right, &uidx)); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); @@ -1217,20 +1062,19 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g } -int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { +int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint16_t flag = FLD_DESC_NO_FREE; + uint16_t flag = 0; filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); if (u->right.type == FLD_TYPE_VALUE) { void *data = FILTER_UNIT_VAL_DATA(src, u); if (IS_VAR_DATA_TYPE(type)) { - if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); + if (FILTER_UNIT_OPTR(u) == OP_TYPE_IN) { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); @@ -1252,7 +1096,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); } -int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { +int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) { SFilterUnit *u = &info->units[uidx]; u->compare.optr2 = optr; @@ -1262,9 +1106,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ } -int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { +int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { SFilterFieldId left, right, right2; - uint16_t uidx = 0; + uint32_t uidx = 0; SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); @@ -1272,17 +1116,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { if (ctx->isnull) { assert(ctx->notnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (ctx->notnull) { assert(ctx->isnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1304,7 +1148,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { @@ -1315,8 +1159,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1326,7 +1170,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1334,7 +1178,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1349,7 +1193,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan assert(ctx->isnull || ctx->notnull || ctx->isrange); if (ctx->isnull) { - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); taosArrayPush(res, g); } @@ -1358,7 +1202,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan assert(!ctx->isrange); memset(g, 0, sizeof(*g)); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); taosArrayPush(res, g); } @@ -1380,7 +1224,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { void *data = malloc(sizeof(int64_t)); @@ -1390,8 +1234,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); filterAddUnitToGroup(g, uidx); } @@ -1406,7 +1250,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1414,7 +1258,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1442,61 +1286,84 @@ static void filterFreeGroup(void *pItem) { } -int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { +EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { int32_t code = TSDB_CODE_SUCCESS; - SArray* leftGroup = NULL; - SArray* rightGroup = NULL; - - if (tree->nodeType != TEXPR_BINARYEXPR_NODE) { - //qError("invalid nodeType:%d", tree->nodeType); - return TSDB_CODE_QRY_APP_ERROR; + SArray* preGroup = NULL; + SArray* newGroup = NULL; + SArray* resGroup = NULL; + ENodeType nType = nodeType(pNode); + SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext; + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + SLogicConditionNode *node = (SLogicConditionNode *)pNode; + if (LOGIC_COND_TYPE_AND == node->condType) { + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + newGroup = taosArrayInit(4, sizeof(SFilterGroup)); + resGroup = taosArrayInit(4, sizeof(SFilterGroup)); + + SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; + nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + FLT_ERR_JRET(tctx.code); + + FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); + + taosArrayDestroyEx(newGroup, filterFreeGroup); + newGroup = NULL; + taosArrayDestroyEx(preGroup, filterFreeGroup); + + preGroup = resGroup; + resGroup = NULL; + + cell = cell->pNext; + } + + taosArrayAddAll(ctx->group, preGroup); + + taosArrayDestroy(preGroup); + + return DEAL_RES_IGNORE_CHILD; + } + + if (LOGIC_COND_TYPE_OR == node->condType) { + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + FLT_ERR_JRET(ctx->code); + + cell = cell->pNext; + } + + return DEAL_RES_IGNORE_CHILD; + } + + fltError("invalid condition type, type:%d", node->condType); + + return DEAL_RES_ERROR; } - - if (tree->_node.optr == TSDB_RELATION_AND) { - leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); - rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); - ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); + if (QUERY_NODE_OPERATOR == nType) { + FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group)); - return TSDB_CODE_SUCCESS; + return DEAL_RES_IGNORE_CHILD; } - if (tree->_node.optr == TSDB_RELATION_OR) { - ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); - ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); - - return TSDB_CODE_SUCCESS; - } - - code = filterAddGroupUnitFromNode(info, tree, group); + fltError("invalid node type for filter, type:%d", nodeType(pNode)); + code = TSDB_CODE_QRY_INVALID_INPUT; _return: - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return code; + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + taosArrayDestroyEx(resGroup, filterFreeGroup); + + ctx->code = code; + + return DEAL_RES_ERROR; } -#if 0 -int32_t filterInitUnitFunc(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - - info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - } - - return TSDB_CODE_SUCCESS; -} -#endif - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { +int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { int32_t n = 0; switch (type) { @@ -1574,34 +1441,38 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { if (qDebugFlag & DEBUG_DEBUG) { -// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); + if (info == NULL) { + fltDebug("%s - FilterInfo: EMPTY", msg); + return; + } if (options == 0) { -// //qDebug("%s - FilterInfo:", msg); -// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + qDebug("%s - FilterInfo:", msg); + qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema *sch = field->desc; -// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + SColumnRefNode *refNode = (SColumnRefNode *)field->desc; + qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); } - //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { + qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); + for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { - SVariant *var = field->desc; - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); + SValueNode *var = (SValueNode *)field->desc; + SDataType *dType = &var->node.resType; + if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { + qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); } else { - //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, dType->type, var->datum.i); //TODO } } else if (field->data) { - //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO } } - //qDebug("UNIT Num:%u", info->unitNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { + qDebug("UNIT Num:%u", info->unitNum); + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; int32_t type = FILTER_UNIT_DATA_TYPE(unit); int32_t len = 0; @@ -1609,17 +1480,19 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SSchema *sch = left->desc; - len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + SColumnRefNode *refNode = (SColumnRefNode *)left->desc; + if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ + len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); + } - if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); char *data = right->data; if (IS_VAR_DATA_TYPE(type)) { tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); + if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); } else { strcat(str, "NULL"); } @@ -1627,32 +1500,34 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS){ + sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); + } - if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); char *data = right->data; if (IS_VAR_DATA_TYPE(type)) { tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); } else { strcat(str, "NULL"); } strcat(str, "]"); } - //qDebug("%s", str); //TODO + qDebug("%s", str); //TODO } - //qDebug("GROUP Num:%u", info->groupNum); - for (uint16_t i = 0; i < info->groupNum; ++i) { + qDebug("GROUP Num:%u", info->groupNum); + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup *group = &info->groups[i]; - //qDebug("Group%d : unit num[%u]", i, group->unitNum); + qDebug("Group%d : unit num[%u]", i, group->unitNum); - for (uint16_t u = 0; u < group->unitNum; ++u) { - //qDebug("unit id:%u", group->unitIdxs[u]); + for (uint32_t u = 0; u < group->unitNum; ++u) { + qDebug("unit id:%u", group->unitIdxs[u]); } } @@ -1660,12 +1535,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) } if (options == 1) { - //qDebug("%s - RANGE info:", msg); + qDebug("%s - RANGE info:", msg); - //qDebug("RANGE Num:%u", info->colRangeNum); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + qDebug("RANGE Num:%u", info->colRangeNum); + for (uint32_t i = 0; i < info->colRangeNum; ++i) { SFilterRangeCtx *ctx = info->colRange[i]; - //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); if (ctx->isrange) { SFilterRangeNode *r = ctx->rs; while (r) { @@ -1675,7 +1550,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str,"(NULL)"); } else { FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); } strcat(str, " - "); @@ -1683,10 +1558,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str, "(NULL)"); } else { FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); } - //qDebug("range: %s", str); + qDebug("range: %s", str); r = r->next; } @@ -1696,25 +1571,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) return; } - //qDebug("%s - Block Filter info:", msg); + qDebug("%s - Block Filter info:", msg); if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - //qDebug("Flag:%s", "ALL"); + qDebug("Flag:%s", "ALL"); return; } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - //qDebug("Flag:%s", "EMPTY"); + qDebug("Flag:%s", "EMPTY"); return; } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ - //qDebug("Flag:%s", "ACTIVE"); + qDebug("Flag:%s", "ACTIVE"); } - //qDebug("GroupNum:%d", info->blkGroupNum); - uint16_t *unitIdx = info->blkUnits; - for (uint16_t i = 0; i < info->blkGroupNum; ++i) { - //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); - uint16_t unitNum = *(unitIdx++); - for (uint16_t m = 0; m < unitNum; ++m) { - //qDebug("uidx[%d]", *(unitIdx++)); + qDebug("GroupNum:%d", info->blkGroupNum); + uint32_t *unitIdx = info->blkUnits; + for (uint32_t i = 0; i < info->blkGroupNum; ++i) { + qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); + uint32_t unitNum = *(unitIdx++); + for (uint32_t m = 0; m < unitNum; ++m) { + qDebug("uidx[%d]", *(unitIdx++)); } } } @@ -1777,14 +1652,6 @@ void filterFreeField(SFilterField* field, int32_t type) { return; } - if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { - if (type == FLD_TYPE_VALUE) { - taosVariantDestroy(field->desc); - } - - tfree(field->desc); - } - if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { taosHashCleanup(field->data); @@ -1800,21 +1667,23 @@ void filterFreePCtx(SFilterPCtx *pctx) { } void filterFreeInfo(SFilterInfo *info) { - CHK_RETV(info == NULL); + if (info == NULL) { + return; + } tfree(info->cunits); tfree(info->blkUnitRes); tfree(info->blkUnits); for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { - for (uint16_t f = 0; f < info->fields[i].num; ++f) { + for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); } tfree(info->fields[i].fields); } - for (int32_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { filterFreeGroup(&info->groups[i]); } @@ -1826,7 +1695,7 @@ void filterFreeInfo(SFilterInfo *info) { tfree(info->unitFlags); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + for (uint32_t i = 0; i < info->colRangeNum; ++i) { filterFreeRangeCtx(info->colRange[i]); } @@ -1844,15 +1713,15 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { uint8_t optr = FILTER_UNIT_OPTR(unit); switch (optr) { - case TSDB_RELATION_GREATER: - case TSDB_RELATION_GREATER_EQUAL: - unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : OP_TYPE_IS_NOT_NULL; break; - case TSDB_RELATION_LESS: - case TSDB_RELATION_LESS_EQUAL: - unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + unit->compare.optr = (extInfo > 0) ? OP_TYPE_IS_NOT_NULL : FILTER_DUMMY_EMPTY_OPTR; break; - case TSDB_RELATION_EQUAL: + case OP_TYPE_EQUAL: unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; break; default: @@ -1863,11 +1732,11 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { } -int32_t filterInitValFieldData(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { +int32_t fltInitValFieldData(SFilterInfo *info) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + assert(unit->compare.optr == OP_TYPE_IS_NULL || unit->compare.optr == OP_TYPE_IS_NOT_NULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); continue; } @@ -1878,53 +1747,73 @@ int32_t filterInitValFieldData(SFilterInfo *info) { uint32_t type = FILTER_UNIT_DATA_TYPE(unit); SFilterField* fi = right; - SVariant* var = fi->desc; + SValueNode* var = (SValueNode *)fi->desc; if (var == NULL) { assert(fi->data != NULL); continue; } - if (unit->compare.optr == TSDB_RELATION_IN) { - filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); -// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + if (unit->compare.optr == OP_TYPE_IN) { + FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); + if (fi->data == NULL) { + fltError("failed to convert in param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); continue; } + SDataType *dType = &var->node.resType; + if (type == TSDB_DATA_TYPE_BINARY) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); } else if (type == TSDB_DATA_TYPE_NCHAR) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - } else { - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE - fi->data = calloc(var->nLen, tDataTypes[type].bytes); - for (int32_t a = 0; a < var->nLen; ++a) { + } else if (type != TSDB_DATA_TYPE_JSON){ + if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE +/* + fi->data = calloc(dType->bytes, tDataTypes[type].bytes); + for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); } - +*/ continue; } else { fi->data = calloc(1, sizeof(int64_t)); } + } else{ // type == TSDB_DATA_TYPE_JSON + // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) } - bool converted = false; - char extInfo = 0; -// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { -// if (converted) { -// filterHandleValueExtInfo(unit, extInfo); -// -// continue; -// } -// //qError("dump value to type[%d] failed", type); -// return TSDB_CODE_TSC_INVALID_OPERATION; -// } + if(type != TSDB_DATA_TYPE_JSON){ + bool converted = false; + char extInfo = 0; + SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; + SScalarParam out = {.data = fi->data, .num = 1, .type = type}; + if (vectorConvertImpl(&in, &out)) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + // match/nmatch for nchar type need convert from ucs4 to mbs + if(type == TSDB_DATA_TYPE_NCHAR && + (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){ + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + if (len < 0){ + qError("filterInitValFieldData taosUcs4ToMbs error 1"); + return TSDB_CODE_FAILED; + } + varDataSetLen(newValData, len); + varDataCopy(fi->data, newValData); + } } return TSDB_CODE_SUCCESS; @@ -1935,34 +1824,40 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) int32_t ret = func(left, right); switch (optr) { - case TSDB_RELATION_EQUAL: { + case OP_TYPE_EQUAL: { return ret == 0; } - case TSDB_RELATION_NOT_EQUAL: { + case OP_TYPE_NOT_EQUAL: { return ret != 0; } - case TSDB_RELATION_GREATER_EQUAL: { + case OP_TYPE_GREATER_EQUAL: { return ret >= 0; } - case TSDB_RELATION_GREATER: { + case OP_TYPE_GREATER_THAN: { return ret > 0; } - case TSDB_RELATION_LESS_EQUAL: { + case OP_TYPE_LOWER_EQUAL: { return ret <= 0; } - case TSDB_RELATION_LESS: { + case OP_TYPE_LOWER_THAN: { return ret < 0; } - case TSDB_RELATION_LIKE: { + case OP_TYPE_LIKE: { return ret == 0; } - case TSDB_RELATION_MATCH: { + case OP_TYPE_NOT_LIKE: { return ret == 0; } - case TSDB_RELATION_NMATCH: { + case OP_TYPE_MATCH: { return ret == 0; } - case TSDB_RELATION_IN: { + case OP_TYPE_NMATCH: { + return ret == 0; + } + case OP_TYPE_IN: { + return ret == 1; + } + case OP_TYPE_NOT_IN: { return ret == 1; } @@ -1982,25 +1877,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c int64_t tmp = 0; switch (uoptr) { - case TSDB_RELATION_GREATER: + case OP_TYPE_GREATER_THAN: SIMPLE_COPY_VALUES(&ra.s, val); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_GREATER_EQUAL: + case OP_TYPE_GREATER_EQUAL: SIMPLE_COPY_VALUES(&ra.s, val); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_LESS: + case OP_TYPE_LOWER_THAN: SIMPLE_COPY_VALUES(&ra.e, val); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_LESS_EQUAL: + case OP_TYPE_LOWER_EQUAL: SIMPLE_COPY_VALUES(&ra.e, val); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_NOT_EQUAL: + case OP_TYPE_NOT_EQUAL: assert(type == TSDB_DATA_TYPE_BOOL); if (GET_INT8_VAL(val)) { SIMPLE_COPY_VALUES(&ra.s, &tmp); @@ -2011,7 +1906,7 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c SIMPLE_COPY_VALUES(&ra.e, &tmp); } break; - case TSDB_RELATION_EQUAL: + case OP_TYPE_EQUAL: SIMPLE_COPY_VALUES(&ra.s, val); SIMPLE_COPY_VALUES(&ra.e, val); break; @@ -2025,25 +1920,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c } int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { - CHK_JMP(ctx1->status != ctx2->status); - CHK_JMP(ctx1->isnull != ctx2->isnull); - CHK_JMP(ctx1->notnull != ctx2->notnull); - CHK_JMP(ctx1->isrange != ctx2->isrange); + FLT_CHK_JMP(ctx1->status != ctx2->status); + FLT_CHK_JMP(ctx1->isnull != ctx2->isnull); + FLT_CHK_JMP(ctx1->notnull != ctx2->notnull); + FLT_CHK_JMP(ctx1->isrange != ctx2->isrange); SFilterRangeNode *r1 = ctx1->rs; SFilterRangeNode *r2 = ctx2->rs; while (r1 && r2) { - CHK_JMP(r1->ra.sflag != r2->ra.sflag); - CHK_JMP(r1->ra.eflag != r2->ra.eflag); - CHK_JMP(r1->ra.s != r2->ra.s); - CHK_JMP(r1->ra.e != r2->ra.e); + FLT_CHK_JMP(r1->ra.sflag != r2->ra.sflag); + FLT_CHK_JMP(r1->ra.eflag != r2->ra.eflag); + FLT_CHK_JMP(r1->ra.s != r2->ra.s); + FLT_CHK_JMP(r1->ra.e != r2->ra.e); r1 = r1->next; r2 = r2->next; } - CHK_JMP(r1 != r2); + FLT_CHK_JMP(r1 != r2); *equal = true; @@ -2055,7 +1950,7 @@ _return: } -int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { +int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) { SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; int32_t size = (int32_t)taosArrayGetSize(colArray); int32_t type = gRes->colInfo[colIdx].dataType; @@ -2065,12 +1960,16 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI SFilterUnit* u = taosArrayGetP(colArray, i); uint8_t optr = FILTER_UNIT_OPTR(u); - filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); - CHK_JMP(*empty); + filterAddRangeOptr(ctx, optr, LOGIC_COND_TYPE_AND, empty, NULL); + FLT_CHK_JMP(*empty); if (!FILTER_NO_MERGE_OPTR(optr)) { - filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); - CHK_JMP(MR_EMPTY_RES(ctx)); + filterAddUnitRange(info, u, ctx, LOGIC_COND_TYPE_AND); + FLT_CHK_JMP(MR_EMPTY_RES(ctx)); + } + if(FILTER_UNIT_OPTR(u) == OP_TYPE_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ + gRes->colInfo[colIdx].optr = OP_TYPE_EQUAL; + SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u)); } } @@ -2092,11 +1991,11 @@ _return: int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { bool empty = false; - uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); - uint16_t colIdxi = 0; - uint16_t gResIdx = 0; + uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); + uint32_t colIdxi = 0; + uint32_t gResIdx = 0; - for (uint16_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup* g = info->groups + i; gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); @@ -2104,9 +2003,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t colIdxi = 0; empty = false; - for (uint16_t j = 0; j < g->unitNum; ++j) { + for (uint32_t j = 0; j < g->unitNum; ++j) { SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); - uint16_t cidx = FILTER_UNIT_COL_IDX(u); + uint32_t cidx = FILTER_UNIT_COL_IDX(u); if (gRes[gResIdx]->colInfo[cidx].info == NULL) { gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); @@ -2122,10 +2021,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } if (colIdxi > 1) { - qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); } - for (uint16_t l = 0; l < colIdxi; ++l) { + for (uint32_t l = 0; l < colIdxi; ++l) { int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; if (FILTER_NO_MERGE_DATA_TYPE(type)) { @@ -2164,7 +2063,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool equal = false; for (; m < gRes1->colNum; ++m) { @@ -2187,6 +2086,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict = true; return; } + + // for long in operation + if (gRes1->colInfo[idx1].optr == OP_TYPE_EQUAL && gRes2->colInfo[idx2].optr == OP_TYPE_EQUAL) { + SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info; + if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){ + *conflict = true; + return; + } + } ++n; equal = true; @@ -2204,7 +2112,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool } -int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { +int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); @@ -2234,10 +2142,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; bool equal = false; - uint16_t equal1 = 0, equal2 = 0, merNum = 0; + uint32_t equal1 = 0, equal2 = 0, merNum = 0; SFilterRangeCtx *ctx = NULL; SFilterColCtx colCtx = {0}; SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); @@ -2256,9 +2164,9 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++merNum; - filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); + filterMergeTwoGroupsImpl(info, &ctx, LOGIC_COND_TYPE_OR, idx1, *gRes1, *gRes2, NULL, all); - CHK_JMP(*all); + FLT_CHK_JMP(*all); if (numEqual) { if ((*gRes1)->colNum == 1) { @@ -2278,7 +2186,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++equal2; } - CHK_JMP(equal1 != merNum && equal2 != merNum); + FLT_CHK_JMP(equal1 != merNum && equal2 != merNum); colCtx.colIdx = idx1; colCtx.ctx = ctx; ctx = NULL; @@ -2290,7 +2198,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++equal1; } - CHK_JMP(equal1 != merNum); + FLT_CHK_JMP(equal1 != merNum); colCtx.colIdx = idx1; colCtx.ctx = ctx; ctx = NULL; @@ -2351,7 +2259,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); int32_t pEnd = 0, cStart = 0, cEnd = 0; - uint16_t pColNum = 0, cColNum = 0; + uint32_t pColNum = 0, cColNum = 0; int32_t movedNum = 0; bool all = false; @@ -2371,7 +2279,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR assert(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - CHK_JMP(all); + FLT_CHK_JMP(all); if (gRes[n] == NULL) { if (n < ((*gResNum) - 1)) { @@ -2392,7 +2300,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR assert(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - CHK_JMP(all); + FLT_CHK_JMP(all); if (gRes[n] == NULL) { if (n < ((*gResNum) - 1)) { @@ -2417,7 +2325,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR } cStart = i; - cEnd = i; cColNum = gRes[i]->colNum; } @@ -2433,7 +2340,7 @@ _return: int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { size_t groupSize = taosArrayGetSize(group); - info->groupNum = (uint16_t)groupSize; + info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { info->groups = calloc(info->groupNum, sizeof(*info->groups)); @@ -2450,7 +2357,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { - //qDebug("no need rewrite"); + qDebug("no need rewrite"); return TSDB_CODE_SUCCESS; } @@ -2462,7 +2369,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum SFilterGroupCtx *res = NULL; SFilterColInfo *colInfo = NULL; int32_t optr = 0; - uint16_t uidx = 0; + uint32_t uidx = 0; memset(info, 0, sizeof(*info)); @@ -2471,18 +2378,18 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum oinfo.colRangeNum = 0; oinfo.colRange = NULL; - FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); + FILTER_SET_FLAG(info->options, FLT_OPTION_NEED_UNIQE); filterInitUnitsFields(info); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + optr = (res->colNum > 1) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR; SFilterGroup ng = {0}; - for (uint16_t m = 0; m < res->colNum; ++m) { + for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { assert(colInfo->type == RANGE_TYPE_UNIT); @@ -2518,13 +2425,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum } int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - uint16_t *idxs = NULL; - uint16_t colNum = 0; + uint32_t *idxs = NULL; + uint32_t colNum = 0; SFilterGroupCtx *res = NULL; - uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); for (int32_t i = 0; i < gResNum; ++i) { - for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { + for (uint32_t m = 0; m < gRes[i]->colNum; ++m) { SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { continue; @@ -2534,7 +2441,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ } } - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { if (idxNum[i] < gResNum) { continue; } @@ -2548,16 +2455,16 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ idxs[colNum++] = i; } - CHK_JMP(colNum <= 0); + FLT_CHK_JMP(colNum <= 0); info->colRangeNum = colNum; info->colRange = calloc(colNum, POINTER_BYTES); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - uint16_t n = 0; + uint32_t n = 0; - for (uint16_t m = 0; m < info->colRangeNum; ++m) { + for (uint32_t m = 0; m < info->colRangeNum; ++m) { for (; n < res->colNum; ++n) { if (res->colIdx[n] < idxs[m]) { continue; @@ -2569,13 +2476,13 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ if (info->colRange[m] == NULL) { info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); - info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; + info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi); } assert(colInfo->type == RANGE_TYPE_MR_CTX); bool all = false; - filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); + filterSourceRangeFromCtx(info->colRange[m], colInfo->info, LOGIC_COND_TYPE_OR, NULL, &all); if (all) { filterFreeRangeCtx(info->colRange[m]); info->colRange[m] = NULL; @@ -2588,7 +2495,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ --info->colRangeNum; --m; - CHK_JMP(info->colRangeNum <= 0); + FLT_CHK_JMP(info->colRangeNum <= 0); } ++n; @@ -2605,7 +2512,7 @@ _return: } int32_t filterPostProcessRange(SFilterInfo *info) { - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + for (uint32_t i = 0; i < info->colRangeNum; ++i) { SFilterRangeCtx* ctx = info->colRange[i]; SFilterRangeNode *r = ctx->rs; while (r) { @@ -2619,13 +2526,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) { - uint16_t n = 0; - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); @@ -2635,7 +2540,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant + info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); + }else{ + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } } else { info->cunits[i].valData = NULL; } @@ -2648,30 +2557,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); } - - uint16_t cgroupNum = info->groupNum + 1; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - cgroupNum += info->groups[i].unitNum; - } - - info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); - - for (uint16_t i = 0; i < info->groupNum; ++i) { - info->cgroups[n++] = info->groups[i].unitNum; - - for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { - info->cgroups[n++] = info->groups[i].unitIdxs[m]; - } - } - - info->cgroups[n] = 0; return TSDB_CODE_SUCCESS; } int32_t filterUpdateComUnits(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); @@ -2686,7 +2577,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); - for (int32_t k = 0; k < info->unitNum; ++k) { + for (uint32_t k = 0; k < info->unitNum; ++k) { int32_t index = -1; SFilterComUnit *cunit = &info->cunits[k]; @@ -2706,20 +2597,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } if (pDataStatis[index].numOfNull <= 0) { - if (cunit->optr == TSDB_RELATION_ISNULL) { + if (cunit->optr == OP_TYPE_IS_NULL) { info->blkUnitRes[k] = -1; rmUnit = 1; continue; } - if (cunit->optr == TSDB_RELATION_NOTNULL) { + if (cunit->optr == OP_TYPE_IS_NOT_NULL) { info->blkUnitRes[k] = 1; rmUnit = 1; continue; } } else { if (pDataStatis[index].numOfNull == numOfRows) { - if (cunit->optr == TSDB_RELATION_ISNULL) { + if (cunit->optr == OP_TYPE_IS_NULL) { info->blkUnitRes[k] = 1; rmUnit = 1; continue; @@ -2731,18 +2622,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } } - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL - || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL) { + if (cunit->optr == OP_TYPE_IS_NULL || cunit->optr == OP_TYPE_IS_NOT_NULL + || cunit->optr == OP_TYPE_IN || cunit->optr == OP_TYPE_LIKE || cunit->optr == OP_TYPE_MATCH + || cunit->optr == OP_TYPE_NOT_EQUAL) { continue; } SColumnDataAgg* pDataBlockst = &pDataStatis[index]; void *minVal, *maxVal; + float minv = 0; + float maxv = 0; if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); + minv = (float)(*(double *)(&pDataBlockst->min)); + maxv = (float)(*(double *)(&pDataBlockst->max)); minVal = &minv; maxVal = &maxv; @@ -2761,8 +2654,8 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 info->blkUnitRes[k] = 1; rmUnit = 1; } else if ((!minRes) && (!maxRes)) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); + minRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_LOWER_EQUAL, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_GREATER_EQUAL, maxVal, cunit->valData2); if (minRes && maxRes) { continue; @@ -2779,9 +2672,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 info->blkUnitRes[k] = 1; rmUnit = 1; } else if ((!minRes) && (!maxRes)) { - if (cunit->optr == TSDB_RELATION_EQUAL) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); + if (cunit->optr == OP_TYPE_EQUAL) { + minRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_GREATER_THAN, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_LOWER_THAN, maxVal, cunit->valData); if (minRes || maxRes) { info->blkUnitRes[k] = -1; rmUnit = 1; @@ -2797,12 +2690,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } -// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); + if (rmUnit == 0) { + fltDebug("NO Block Filter APPLY"); + FLT_RET(TSDB_CODE_SUCCESS); + } info->blkGroupNum = info->groupNum; - uint16_t *unitNum = info->blkUnits; - uint16_t *unitIdx = unitNum + 1; + uint32_t *unitNum = info->blkUnits; + uint32_t *unitIdx = unitNum + 1; int32_t all = 0, empty = 0; for (uint32_t g = 0; g < info->groupNum; ++g) { @@ -2812,7 +2708,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 empty = 0; for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + uint32_t uidx = group->unitIdxs[u]; if (info->blkUnitRes[uidx] == 1) { --(*unitNum); all = 1; @@ -2859,17 +2755,19 @@ _return: bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - uint16_t *unitIdx = NULL; - - *p = calloc(numOfRows, sizeof(int8_t)); + uint32_t *unitIdx = NULL; + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); unitIdx = info->blkUnits; for (uint32_t g = 0; g < info->blkGroupNum; ++g) { - uint16_t unitNum = *(unitIdx++); + uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; void *colData = (char *)cunit->colData + cunit->dataSize * i; @@ -2880,11 +2778,11 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint8_t optr = cunit->optr; if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { - if (optr == TSDB_RELATION_NOTNULL) { + if (optr == OP_TYPE_IS_NOT_NULL) { (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { + } else if (optr == OP_TYPE_IS_NULL) { (*p)[i] = 0; } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); @@ -2964,12 +2862,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, return all; } - *p = calloc(numOfRows, sizeof(int8_t)); + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if (!colData){ // for json->'key' is null + (*p)[i] = 1; + }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null + colData = POINTER_SHIFT(colData, CHAR_BYTES); + (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + }else{ + (*p)[i] = 0; + } + }else{ + (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType)); + } if ((*p)[i] == 0) { all = false; } @@ -2985,12 +2896,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } - *p = calloc(numOfRows, sizeof(int8_t)); - + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if (!colData) { // for json->'key' is not null + (*p)[i] = 0; + }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null + colData = POINTER_SHIFT(colData, CHAR_BYTES); + (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + }else{ // for json->'key' is not null + (*p)[i] = 1; + } + }else { + (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType)); + } + if ((*p)[i] == 0) { all = false; } @@ -3013,10 +2939,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD return all; } - *p = calloc(numOfRows, sizeof(int8_t)); + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - if (isNull(colData, info->cunits[0].dataType)) { + if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { all = false; colData += dataSize; continue; @@ -3041,19 +2969,35 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { return all; } - - *p = calloc(numOfRows, sizeof(int8_t)); + + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - if (isNull(colData, info->cunits[uidx].dataType)) { + if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { + (*p)[i] = 0; all = false; continue; } + // match/nmatch for nchar type need convert from ucs4 to mbs + + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ + char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); + } + tfree(newColData); + }else{ + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + } - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - if ((*p)[i] == 0) { all = false; } @@ -3071,15 +3015,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg return all; } - *p = calloc(numOfRows, sizeof(int8_t)); - + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); for (uint32_t g = 0; g < info->groupNum; ++g) { SFilterGroup *group = &info->groups[g]; for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; void *colData = (char *)cunit->colData + cunit->dataSize * i; @@ -3088,17 +3034,29 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg //} else { uint8_t optr = cunit->optr; - if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + if (colData == NULL || isNull(colData, cunit->dataType)) { + (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { - if (optr == TSDB_RELATION_NOTNULL) { + if (optr == OP_TYPE_IS_NOT_NULL) { (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { + } else if (optr == OP_TYPE_IS_NULL) { (*p)[i] = 0; } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ + char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); + } + tfree(newColData); + }else{ + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } } //FILTER_UNIT_SET_R(info, uidx, p[i]); @@ -3123,11 +3081,6 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg return all; } - -FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { - return (*info->func)(info, numOfRows, p, statis, numOfCols); -} - int32_t filterSetExecFunc(SFilterInfo *info) { if (FILTER_ALL_RES(info)) { info->func = filterExecuteImplAll; @@ -3144,12 +3097,12 @@ int32_t filterSetExecFunc(SFilterInfo *info) { return TSDB_CODE_SUCCESS; } - if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { + if (info->units[0].compare.optr == OP_TYPE_IS_NULL) { info->func = filterExecuteImplIsNull; return TSDB_CODE_SUCCESS; } - if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { + if (info->units[0].compare.optr == OP_TYPE_IS_NOT_NULL) { info->func = filterExecuteImplNotNull; return TSDB_CODE_SUCCESS; } @@ -3174,13 +3127,13 @@ int32_t filterPreprocess(SFilterInfo *info) { filterMergeGroups(info, gRes, &gResNum); if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { -// qInfo("Final - FilterInfo: [ALL]"); + fltInfo("Final - FilterInfo: [ALL]"); goto _return; } if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { -// qInfo("Final - FilterInfo: [EMPTY]"); + fltInfo("Final - FilterInfo: [EMPTY]"); goto _return; } @@ -3207,25 +3160,19 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { -// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); -// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); +int32_t fltSetColFieldDataImpl(SFilterInfo *info, void *param, filer_get_col_from_id fp, bool fromColId) { if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { return TSDB_CODE_SUCCESS; } - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema* sch = fi->desc; - - for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); - if (sch->colId == pColInfo->info.colId) { - fi->data = pColInfo->pData; - - break; - } + + if (fromColId) { + (*fp)(param, FILTER_GET_COL_FIELD_ID(fi), &fi->data); + } else { + (*fp)(param, FILTER_GET_COL_FIELD_SLOT_ID(fi), &fi->data); } } @@ -3235,45 +3182,32 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat } -int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { +int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { int32_t code = TSDB_CODE_SUCCESS; - SFilterInfo *info = NULL; - -// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); - - if (*pinfo == NULL) { - *pinfo = calloc(1, sizeof(SFilterInfo)); - } - - info = *pinfo; - - info->options = options; SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); filterInitUnitsFields(info); - code = filterTreeToGroup(tree, info, group); - - ERR_JRET(code); + SFltBuildGroupCtx tctx = {.info = info, .group = group}; + nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); + taosArrayDestroy(group); - ERR_JRET(filterInitValFieldData(info)); + FLT_ERR_JRET(fltInitValFieldData(info)); - if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { + if (!FILTER_GET_FLAG(info->options, FLT_OPTION_NO_REWRITE)) { filterDumpInfoToString(info, "Before preprocess", 0); - ERR_JRET(filterPreprocess(info)); + FLT_ERR_JRET(filterPreprocess(info)); - CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); + FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - taosArrayDestroy(group); return code; } - - //ERR_JRET(filterInitUnitFunc(info)); } info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); @@ -3281,18 +3215,12 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option filterDumpInfoToString(info, "Final", 0); - taosArrayDestroy(group); - return code; _return: -// qInfo("No filter, code:%d", code); + + qInfo("init from node failed, code:%d", code); - taosArrayDestroy(group); - filterFreeInfo(*pinfo); - - *pinfo = NULL; - return code; } @@ -3311,7 +3239,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t bool ret = true; void *minVal, *maxVal; - for (int32_t k = 0; k < info->colRangeNum; ++k) { + for (uint32_t k = 0; k < info->colRangeNum; ++k) { int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for(int32_t i = 0; i < numOfCols; ++i) { @@ -3331,30 +3259,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t break; } - if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { - ret = false; - break; - } - - // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { - - // if isNULL query exists, load the null data column - if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + if (pDataStatis[index].numOfNull <= 0) { + if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } + } else if (pDataStatis[index].numOfNull > 0) { + if (pDataStatis[index].numOfNull == numOfRows) { + if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + ret = false; + break; + } - continue; + continue; + } else { + if (ctx->isnull) { + continue; + } + } } SColumnDataAgg* pDataBlockst = &pDataStatis[index]; SFilterRangeNode *r = ctx->rs; + float minv = 0; + float maxv = 0; if (ctx->type == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); + minv = (float)(*(double *)(&pDataBlockst->min)); + maxv = (float)(*(double *)(&pDataBlockst->max)); minVal = &minv; maxVal = &maxv; @@ -3371,7 +3304,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t r = r->next; } - CHK_RET(!ret, ret); + if (!ret) { + return ret; + } } return ret; @@ -3381,45 +3316,38 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { SFilterRange ra = {0}; - SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); + SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); SFilterRangeCtx *cur = NULL; int32_t num = 0; int32_t optr = 0; int32_t code = 0; bool empty = false, all = false; - for (int32_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup *group = &info->groups[i]; if (group->unitNum > 1) { cur = tmpc; - optr = TSDB_RELATION_AND; + optr = LOGIC_COND_TYPE_AND; } else { cur = prev; - optr = TSDB_RELATION_OR; + optr = LOGIC_COND_TYPE_OR; } - for (int32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint32_t uidx = group->unitIdxs[u]; SFilterUnit *unit = &info->units[uidx]; uint8_t raOptr = FILTER_UNIT_OPTR(unit); - filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); - CHK_JMP(empty); + filterAddRangeOptr(cur, raOptr, LOGIC_COND_TYPE_AND, &empty, NULL); + FLT_CHK_JMP(empty); if (FILTER_NO_MERGE_OPTR(raOptr)) { continue; } - - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - void *s = FILTER_GET_VAL_FIELD_DATA(right); - void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - SIMPLE_COPY_VALUES(&ra.s, s); - SIMPLE_COPY_VALUES(&ra.e, e); - - filterAddRange(cur, &ra, optr); + filterAddUnitRange(info, unit, cur, optr); } if (cur->notnull) { @@ -3428,7 +3356,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { } if (group->unitNum > 1) { - filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); + filterSourceRangeFromCtx(prev, cur, LOGIC_COND_TYPE_OR, &empty, &all); filterResetRangeCtx(cur); if (all) { break; @@ -3441,11 +3369,11 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { } else { filterGetRangeNum(prev, &num); if (num > 1) { - //qError("only one time range accepted, num:%d", num); - ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); + qError("only one time range accepted, num:%d", num); + FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); } - CHK_JMP(num < 1); + FLT_CHK_JMP(num < 1); SFilterRange tra; filterGetRangeRes(prev, &tra); @@ -3456,7 +3384,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { filterFreeRangeCtx(prev); filterFreeRangeCtx(tmpc); - //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); return TSDB_CODE_SUCCESS; _return: @@ -3466,14 +3394,18 @@ _return: filterFreeRangeCtx(prev); filterFreeRangeCtx(tmpc); - //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); return code; } int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { @@ -3486,7 +3418,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar char *src = FILTER_GET_COL_FIELD_DATA(fi, j); char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + char *varSrc = varDataVal(src); + size_t k = 0, varSrcLen = varDataLen(src); + while (k < varSrcLen && varSrc[k++] == -1) {} + if (k == varSrcLen) { + /* NULL */ + varDataLen(dst) = (VarDataLenT) varSrcLen; + varDataCopy(dst, src); + continue; + } + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + if(!ret) { + qError("filterConverNcharColumns taosMbsToUcs4 error"); + return TSDB_CODE_FAILED; + } varDataLen(dst) = len; } @@ -3504,7 +3449,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar } int32_t filterFreeNcharColumns(SFilterInfo* info) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { @@ -3515,7 +3460,198 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { return TSDB_CODE_SUCCESS; } +EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { + SFltTreeStat *stat = (SFltTreeStat *)pContext; + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + fltError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + stat->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + if ((QUERY_NODE_OPERATOR != nodeType(cell->pNode)) && (QUERY_NODE_LOGIC_CONDITION != nodeType(cell->pNode))) { + stat->scalarMode = true; + } + + cell = cell->pNext; + } + + return DEAL_RES_CONTINUE; + } + + if (stat->scalarMode) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + SOperatorNode *node = (SOperatorNode *)*pNode; + if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (NULL == node->pRight) { + if (scalarGetOperatorParamNum(node->opType) > 1) { + fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node)); + stat->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } else { + if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (nodeType(node->pLeft) == nodeType(node->pRight)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + SNode *t = node->pLeft; + node->pLeft = node->pRight; + node->pRight = t; + } + + if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { + fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); + stat->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + if (OP_TYPE_IN != node->opType) { + SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SValueNode *valueNode = (SValueNode *)node->pRight; + int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->dataType.type) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } + } + + return DEAL_RES_CONTINUE; + } + + fltError("invalid node type for filter, type:%d", nodeType(*pNode)); + + stat->code = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; +} + +int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + + FLT_RET(pStat->code); +} + +int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + +} + + +int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { + int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; + SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); + if (id == pColInfo->info.colId) { + *data = pColInfo->pData; + break; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + return fltSetColFieldDataImpl(info, param, fp, false); +} + +int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + return fltSetColFieldDataImpl(info, param, fp, true); +} +int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) { + int32_t code = 0; + SFilterInfo *info = NULL; + + if (pNode == NULL || pInfo == NULL) { + fltError("invalid param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (*pInfo == NULL) { + *pInfo = calloc(1, sizeof(SFilterInfo)); + if (NULL == *pInfo) { + fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + info = *pInfo; + info->options = options; + + SFltTreeStat stat = {0}; + FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + + if (!info->scalarMode) { + FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); + } else { + info->sclCtx.node = pNode; + FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat)); + } + + return code; + +_return: + + filterFreeInfo(*pInfo); + + *pInfo = NULL; + + FLT_RET(code); +} + +bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + if (info->scalarMode) { + SScalarParam output = {0}; + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); + + *p = output.data; + return TSDB_CODE_SUCCESS; + } + + return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); +} + + diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c new file mode 100644 index 0000000000..336ef8f4f8 --- /dev/null +++ b/source/libs/scalar/src/scalar.c @@ -0,0 +1,684 @@ +#include "nodes.h" +#include "common.h" +#include "querynodes.h" +#include "function.h" +#include "functionMgt.h" +#include "sclvector.h" +#include "sclInt.h" + +int32_t scalarGetOperatorParamNum(EOperatorType type) { + if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type + || OP_TYPE_IS_FALSE == type || OP_TYPE_IS_NOT_FALSE == type || OP_TYPE_IS_UNKNOWN == type || OP_TYPE_IS_NOT_UNKNOWN == type) { + return 1; + } + + return 2; +} + +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + if (NULL == pObj) { + sclError("taosHashInit failed, size:%d", 256); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + + int32_t code = 0; + SNodeListNode *nodeList = (SNodeListNode *)pNode; + SListCell *cell = nodeList->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; + int8_t dummy = 0; + int32_t bufLen = 60; + out.data = malloc(bufLen); + int32_t len = 0; + void *buf = NULL; + + for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + + if (valueNode->node.resType.type != type) { + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + + code = vectorConvertImpl(&in, &out); + if (code) { + sclError("convert from %d to %d failed", in.type, out.type); + SCL_ERR_JRET(code); + } + + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(out.data); + buf = varDataVal(out.data); + } else { + len = tDataTypes[type].bytes; + buf = out.data; + } + } else { + buf = nodesGetValueFromNode(valueNode); + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(buf); + buf = varDataVal(buf); + } else { + len = valueNode->node.resType.bytes; + buf = out.data; + } + } + + if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { + sclError("taosHashPut failed"); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + cell = cell->pNext; + } + + tfree(out.data); + *data = pObj; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(out.data); + taosHashCleanup(pObj); + + SCL_RET(code); +} + + +void sclFreeRes(SHashObj *res) { + SScalarParam *p = NULL; + void *pIter = taosHashIterate(res, NULL); + while (pIter) { + p = (SScalarParam *)pIter; + + if (p) { + tfree(p->data); + } + + pIter = taosHashIterate(res, pIter); + } + + taosHashCleanup(res); +} + +void sclFreeParam(SScalarParam *param) { + tfree(param->data); +} + +int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { + switch (nodeType(node)) { + case QUERY_NODE_VALUE: { + SValueNode *valueNode = (SValueNode *)node; + param->data = nodesGetValueFromNode(valueNode); + param->num = 1; + param->type = valueNode->node.resType.type; + param->bytes = valueNode->node.resType.bytes; + param->colData = false; + + break; + } + case QUERY_NODE_NODE_LIST: { + SNodeListNode *nodeList = (SNodeListNode *)node; + if (nodeList->pNodeList->length <= 0) { + sclError("invalid length in nodeList, length:%d", nodeList->pNodeList->length); + SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); + param->num = 1; + param->type = SCL_DATA_TYPE_DUMMY_HASH; + param->colData = false; + + break; + } + case QUERY_NODE_COLUMN_REF: { + if (NULL == ctx) { + sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SColumnRefNode *ref = (SColumnRefNode *)node; + if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { + sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); + if (IS_VAR_DATA_TYPE(columnData->info.type)) { + param->data = columnData; + param->colData = true; + } else { + param->data = columnData->pData; + param->colData = false; + } + + param->num = ctx->pSrc->info.rows; + param->type = columnData->info.type; + param->bytes = columnData->info.bytes; + + break; + } + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_OPERATOR: { + if (NULL == ctx) { + sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); + if (NULL == res) { + sclError("no result for node, type:%d, node:%p", nodeType(node), node); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *param = *res; + + break; + } + } + + if (param->num > *rowNum) { + if ((1 != param->num) && (1 < *rowNum)) { + sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->num); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + *rowNum = param->num; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { + SScalarParam *param = NULL; + + for (int32_t i = 0; i < num; ++i) { + param = params + i; + + if (1 == param->num) { + continue; + } + + if (IS_VAR_DATA_TYPE(param->type)) { + param->data = (char *)(param->data) + varDataTLen(param->data); + } else { + param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { + int32_t code = 0; + SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == paramList) { + sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SListCell *cell = pParamList->pHead; + for (int32_t i = 0; i < pParamList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SCL_ERR_JRET(sclInitParam(cell->pNode, ¶mList[i], ctx, rowNum)); + + cell = cell->pNext; + } + + *pParams = paramList; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(paramList); + SCL_RET(code); +} + +int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) { + int32_t code = 0; + int32_t paramNum = scalarGetOperatorParamNum(node->opType); + if (NULL == node->pLeft || (paramNum == 2 && NULL == node->pRight)) { + sclError("invalid operation node, left:%p, right:%p", node->pLeft, node->pRight); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam)); + if (NULL == paramList) { + sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); + if (paramNum > 1) { + SCL_ERR_JRET(sclInitParam(node->pRight, ¶mList[1], ctx, rowNum)); + } + + *pParams = paramList; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(paramList); + SCL_RET(code); +} + + +int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarFuncExecFuncs ffpSet = {0}; + int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); + if (code) { + sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_RET(code); + } + + SScalarParam *params = NULL; + int32_t rowNum = 0; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < rowNum; ++i) { + code = (*ffpSet.process)(params, node->pParameterList->length, output); + if (code) { + sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } + + sclParamMoveNext(output, 1); + sclParamMoveNext(params, node->pParameterList->length); + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + + +int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *output) { + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (TSDB_DATA_TYPE_BOOL != node->node.resType.type) { + sclError("invalid logic resType, type:%d", node->node.resType.type); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (LOGIC_COND_TYPE_NOT == node->condType && node->pParameterList->length > 1) { + sclError("invalid NOT operation parameter number, paramNum:%d", node->pParameterList->length); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarParam *params = NULL; + int32_t rowNum = 0; + int32_t code = 0; + + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->bytes = sizeof(bool); + output->num = rowNum; + output->data = calloc(rowNum, sizeof(bool)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + void *data = output->data; + + bool value = false; + + for (int32_t i = 0; i < rowNum; ++i) { + for (int32_t m = 0; m < node->pParameterList->length; ++m) { + GET_TYPED_DATA(value, bool, params[m].type, params[m].data); + + if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + break; + } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + break; + } else if (LOGIC_COND_TYPE_NOT == node->condType) { + value = !value; + } + } + + *(bool *)output->data = value; + + sclParamMoveNext(output, 1); + sclParamMoveNext(params, node->pParameterList->length); + } + + output->data = data; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + +int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *output) { + SScalarParam *params = NULL; + int32_t rowNum = 0; + int32_t code = 0; + + SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->num = rowNum; + output->bytes = tDataTypes[output->type].bytes; + output->data = calloc(rowNum, tDataTypes[output->type].bytes); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); + + int32_t paramNum = scalarGetOperatorParamNum(node->opType); + SScalarParam* pLeft = ¶ms[0]; + SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; + + OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + + +EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { + SFunctionNode *node = (SFunctionNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecFuncion(node, NULL, &output); + if (*(int32_t *)pContext) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + + return DEAL_RES_CONTINUE; +} + +EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecLogic(node, NULL, &output); + if (*(int32_t *)pContext) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + + return DEAL_RES_CONTINUE; +} + +EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { + SOperatorNode *node = (SOperatorNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecOperator(node, NULL, &output); + if (*(int32_t *)pContext) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + return sclRewriteFunction(pNode, pContext); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + return sclRewriteLogic(pNode, pContext); + } + + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclRewriteOperator(pNode, pContext); + } + + sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); + + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; +} + + +EDealRes sclWalkFunction(SNode* pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SFunctionNode *node = (SFunctionNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecFuncion(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclWalkLogic(SNode* pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SLogicConditionNode *node = (SLogicConditionNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecLogic(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclWalkOperator(SNode* pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SOperatorNode *node = (SOperatorNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecOperator(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclCalcWalker(SNode* pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + return sclWalkFunction(pNode, pContext); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + return sclWalkLogic(pNode, pContext); + } + + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + return sclWalkOperator(pNode, pContext); + } + + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); + + SScalarCtx *ctx = (SScalarCtx *)pContext; + + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; +} + + + +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { + if (NULL == pNode) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + + nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code); + + if (code) { + nodesDestroyNode(pNode); + SCL_ERR_RET(code); + } + + *pRes = pNode; + + SCL_RET(code); +} + +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { + if (NULL == pNode || NULL == pSrc || NULL == pDst) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + SScalarCtx ctx = {.code = 0, .pSrc = pSrc}; + + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); + + if (ctx.code) { + nodesDestroyNode(pNode); + sclFreeRes(ctx.pRes); + SCL_ERR_RET(ctx.code); + } + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (NULL == res) { + sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode)); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *pDst = *res; + + nodesDestroyNode(pNode); + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/scalar/src/sclfunc.c similarity index 91% rename from source/libs/function/src/tscalarfunction.c rename to source/libs/scalar/src/sclfunc.c index 90d25b3e4a..fd900afcda 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1,14 +1,13 @@ -#include "tscalarfunction.h" -#include "tbinoperator.h" -#include "tunaryoperator.h" +#include "sclfunc.h" +#include "sclvector.h" -static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { +static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { dst->type = src->type; dst->bytes = src->bytes; dst->num = src->num; } -static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tceil(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -34,7 +33,7 @@ static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun } } -static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tfloor(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -62,7 +61,7 @@ static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu } } -static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void _tabs(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -120,7 +119,7 @@ static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun } } -static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tround(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -146,7 +145,7 @@ static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu } } -static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assert(numOfInput == 1); int64_t* out = (int64_t*) pOutput->data; @@ -157,7 +156,7 @@ static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF } } -static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assert(numOfInput > 0); int32_t rowLen = 0; @@ -189,11 +188,11 @@ static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF } } -static void tltrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { } -static void trtrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { } @@ -262,17 +261,15 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -static void setScalarFuncParam(SScalarFuncParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { +static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { param->bytes = bytes; param->type = type; param->num = numOfRows; param->data = pInput; } -bool isStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} +#if 0 int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)) { if (pExprs == NULL) { @@ -361,6 +358,8 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa return 0; } +#endif + SScalarFunctionInfo scalarFunc[8] = { {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c new file mode 100644 index 0000000000..85af663313 --- /dev/null +++ b/source/libs/scalar/src/sclvector.c @@ -0,0 +1,1547 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "ttypes.h" +#include "sclvector.h" +#include "tcompare.h" +#include "querynodes.h" +#include "filterInt.h" +#include "query.h" +#include "sclInt.h" +#include "tep.h" +#include "filter.h" + +//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); + +void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { + int32_t *pLeft = (int32_t *)left; + int32_t *pRight = (int32_t *)right; + double * pOutput = (double *)output; + + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; + + if (numLeft == numRight) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[i] + pRight[i]; + } + } else if (numLeft == 1) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[0] + pRight[i]; + } + } else if (numRight == 1) { + for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + *pOutput = (double)pLeft[i] + pRight[0]; + } + } +} + +typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); + +double getVectorDoubleValue_TINYINT(void *src, int32_t index) { + return (double)*((int8_t *)src + index); +} +double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { + return (double)*((uint8_t *)src + index); +} +double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { + return (double)*((int16_t *)src + index); +} +double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { + return (double)*((uint16_t *)src + index); +} +double getVectorDoubleValue_INT(void *src, int32_t index) { + return (double)*((int32_t *)src + index); +} +double getVectorDoubleValue_UINT(void *src, int32_t index) { + return (double)*((uint32_t *)src + index); +} +double getVectorDoubleValue_BIGINT(void *src, int32_t index) { + return (double)*((int64_t *)src + index); +} +double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { + return (double)*((uint64_t *)src + index); +} +double getVectorDoubleValue_FLOAT(void *src, int32_t index) { + return (double)*((float *)src + index); +} +double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { + return (double)*((double *)src + index); +} + +_getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { + _getDoubleValue_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorDoubleValue_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorDoubleValue_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorDoubleValue_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorDoubleValue_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorDoubleValue_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorDoubleValue_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorDoubleValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorDoubleValue_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorDoubleValue_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorDoubleValue_DOUBLE; + }else { + assert(0); + } + return p; +} + + + +typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); + +int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { + return (int64_t)*((int8_t *)src + index); +} +int64_t getVectorBigintValue_UTINYINT(void *src, int32_t index) { + return (int64_t)*((uint8_t *)src + index); +} +int64_t getVectorBigintValue_SMALLINT(void *src, int32_t index) { + return (int64_t)*((int16_t *)src + index); +} +int64_t getVectorBigintValue_USMALLINT(void *src, int32_t index) { + return (int64_t)*((uint16_t *)src + index); +} +int64_t getVectorBigintValue_INT(void *src, int32_t index) { + return (int64_t)*((int32_t *)src + index); +} +int64_t getVectorBigintValue_UINT(void *src, int32_t index) { + return (int64_t)*((uint32_t *)src + index); +} +int64_t getVectorBigintValue_BIGINT(void *src, int32_t index) { + return (int64_t)*((int64_t *)src + index); +} +int64_t getVectorBigintValue_UBIGINT(void *src, int32_t index) { + return (int64_t)*((uint64_t *)src + index); +} +int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) { + return (int64_t)*((float *)src + index); +} +int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) { + return (int64_t)*((double *)src + index); +} +_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { + _getBigintValue_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorBigintValue_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorBigintValue_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorBigintValue_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorBigintValue_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorBigintValue_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorBigintValue_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorBigintValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorBigintValue_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorBigintValue_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorBigintValue_DOUBLE; + }else { + assert(0); + } + return p; +} + + + + +typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index); + +void* getVectorValueAddr_TINYINT(void *src, int32_t index) { + return (void*)((int8_t *)src + index); +} +void* getVectorValueAddr_UTINYINT(void *src, int32_t index) { + return (void*)((uint8_t *)src + index); +} +void* getVectorValueAddr_SMALLINT(void *src, int32_t index) { + return (void*)((int16_t *)src + index); +} +void* getVectorValueAddr_USMALLINT(void *src, int32_t index) { + return (void*)((uint16_t *)src + index); +} +void* getVectorValueAddr_INT(void *src, int32_t index) { + return (void*)((int32_t *)src + index); +} +void* getVectorValueAddr_UINT(void *src, int32_t index) { + return (void*)((uint32_t *)src + index); +} +void* getVectorValueAddr_BIGINT(void *src, int32_t index) { + return (void*)((int64_t *)src + index); +} +void* getVectorValueAddr_UBIGINT(void *src, int32_t index) { + return (void*)((uint64_t *)src + index); +} +void* getVectorValueAddr_FLOAT(void *src, int32_t index) { + return (void*)((float *)src + index); +} +void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { + return (void*)((double *)src + index); +} +void* getVectorValueAddr_default(void *src, int32_t index) { + return src; +} +void* getVectorValueAddr_VAR(void *src, int32_t index) { + return colDataGet((SColumnInfoData *)src, index); +} + +_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { + _getValueAddr_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorValueAddr_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorValueAddr_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorValueAddr_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorValueAddr_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorValueAddr_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorValueAddr_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorValueAddr_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorValueAddr_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorValueAddr_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorValueAddr_DOUBLE; + }else if(srcType==TSDB_DATA_TYPE_BINARY) { + p = getVectorValueAddr_VAR; + }else if(srcType==TSDB_DATA_TYPE_NCHAR) { + p = getVectorValueAddr_VAR; + }else { + p = getVectorValueAddr_default; + } + return p; +} + + +int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { + int16_t inType = pIn->type; + int16_t inBytes = pIn->bytes; + char *input = pIn->data; + int16_t outType = pOut->type; + int16_t outBytes = pOut->bytes; + char *output = pOut->data; + + switch (outType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + sclError("malloc %d failed", bufSize); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + sclError("calloc %d failed", bufSize); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + sclError("malloc %d failed", bufSize); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + sclError("calloc %d failed", bufSize); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + } + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + sclError("malloc %d failed", bufSize); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + } + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + sclError("calloc %d failed", bufSize); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + } + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + } + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + default: + sclError("invalid convert output type:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { +/* NULL BOOL TINY SMAL INT BIG FLOA DOUB BINA TIME NCHA UTIN USMA UINT UBIG VARC VARB JSON DECI BLOB */ +/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 7, 0, 0, 0, +/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 7, 0, 0, 0, +/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 7, 0, 0, 0, +/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 7, 0, 0, 0, +/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 7, 0, 0, 0, +/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 7, 0, 0, 0, +/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, +/*BINA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, +/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 7, 0, 0, 0, +/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, +/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 7, 0, 0, 0, +/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 7, 0, 0, 0, +/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 7, 0, 0, 0, +/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +int32_t vectorGetConvertType(int32_t type1, int32_t type2) { + if (type1 == type2) { + return 0; + } + + if (type1 < type2) { + return gConvertTypes[type1][type2]; + } + + return gConvertTypes[type2][type1]; +} + +int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) { + if (pLeft->type == pRight->type) { + return TSDB_CODE_SUCCESS; + } + + if (SCL_DATA_TYPE_DUMMY_HASH == pLeft->type || SCL_DATA_TYPE_DUMMY_HASH == pRight->type) { + return TSDB_CODE_SUCCESS; + } + + SScalarParam *param1 = NULL, *paramOut1 = NULL; + SScalarParam *param2 = NULL, *paramOut2 = NULL; + int32_t code = 0; + + if (pLeft->type < pRight->type) { + param1 = pLeft; + param2 = pRight; + paramOut1 = pLeftOut; + paramOut2 = pRightOut; + } else { + param1 = pRight; + param2 = pLeft; + paramOut1 = pRightOut; + paramOut2 = pLeftOut; + } + + + int8_t type = vectorGetConvertType(param1->type, param2->type); + if (0 == type) { + return TSDB_CODE_SUCCESS; + } + + if (type != param1->type) { + paramOut1->bytes = param1->bytes; + paramOut1->type = type; + paramOut1->num = param1->num; + paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); + if (NULL == paramOut1->data) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + code = vectorConvertImpl(param1, paramOut1); + if (code) { + tfree(paramOut1->data); + return code; + } + } + + if (type != param2->type) { + paramOut2->bytes = param2->bytes; + paramOut2->type = type; + paramOut2->num = param2->num; + paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); + if (NULL == paramOut2->data) { + tfree(paramOut1->data); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + code = vectorConvertImpl(param2, paramOut2); + if (code) { + tfree(paramOut1->data); + tfree(paramOut2->data); + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} + +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} + +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + double right = getVectorDoubleValueFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(&right, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} + +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + double * output = (double *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double left = getVectorDoubleValueFnLeft(pLeft->data, i); + double right = getVectorDoubleValueFnRight(pRight->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } else if (pLeft->num == 1) { + double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(&left, pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double right = getVectorDoubleValueFnRight(pRight->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } else if (pRight->num == 1) { + double right = getVectorDoubleValueFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(&right, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double left = getVectorDoubleValueFnLeft(pLeft->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} + +void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t len = pLeft->bytes + pRight->bytes; + + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + char *output = (char *)out; + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += len) { + char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); + char* right = POINTER_SHIFT(pRight->data, pRight->bytes * i); + + if (isNull(left, pLeft->type) || isNull(right, pRight->type)) { + setVardataNull(output, TSDB_DATA_TYPE_BINARY); + continue; + } + + // todo define a macro + memcpy(varDataVal(output), varDataVal(left), varDataLen(left)); + memcpy(varDataVal(output) + varDataLen(left), varDataVal(right), varDataLen(right)); + varDataSetLen(output, varDataLen(left) + varDataLen(right)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += len) { + char *right = POINTER_SHIFT(pRight->data, pRight->bytes * i); + if (isNull(pLeft->data, pLeft->type) || isNull(right, pRight->type)) { + setVardataNull(output, TSDB_DATA_TYPE_BINARY); + continue; + } + + memcpy(varDataVal(output), varDataVal(pLeft->data), varDataLen(pLeft->data)); + memcpy(varDataVal(output) + varDataLen(pLeft->data), varDataVal(right), varDataLen(right)); + varDataSetLen(output, varDataLen(pLeft->data) + varDataLen(right)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += len) { + char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); + if (isNull(left, pLeft->type) || isNull(pRight->data, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + memcpy(varDataVal(output), varDataVal(left), varDataLen(pRight->data)); + memcpy(varDataVal(output) + varDataLen(left), varDataVal(pRight->data), varDataLen(pRight->data)); + varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); + } + } + +} + + +void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + int64_t *output=(int64_t *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + + SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} + +void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + + int64_t *output=(int64_t *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + + SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + } + } + + tfree(leftParam.data); + tfree(rightParam.data); +} + + +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr); + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } + + if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) { + getVectorValueAddrFnRight = getVectorValueAddr_default; + } else { + getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + } + + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i)); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } else if (pLeft->num == 1) { + void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i)); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } else if (pRight->num == 1) { + void *rightData = getVectorValueAddrFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } +} + +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { + SScalarParam pLeftOut = {0}; + SScalarParam pRightOut = {0}; + + vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); + + SScalarParam *param1 = NULL; + SScalarParam *param2 = NULL; + + int32_t type = 0; + if (pLeftOut.type) { + param1 = &pLeftOut; + } else { + param1 = pLeft; + } + + if (pRightOut.type) { + param2 = &pRightOut; + } else { + param2 = pRight; + } + + vectorCompareImpl(param1, param2, out, _ord, optr); +} + +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN); +} + +void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL); +} + +void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN); +} + +void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL); +} + +void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL); +} + +void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL); +} + +void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN); +} + +void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN); +} + +void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE); +} + +void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE); +} + +void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH); +} + +void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH); +} + +void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + res = true; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } +} + +void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + bool res = false; + + bool *output = (bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = true; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } +} + +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + vectorConvertImpl(pLeft, &output); +} + + +_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { + switch (binFunctionId) { + case OP_TYPE_ADD: + return vectorAdd; + case OP_TYPE_SUB: + return vectorSub; + case OP_TYPE_MULTI: + return vectorMultiply; + case OP_TYPE_DIV: + return vectorDivide; + case OP_TYPE_MOD: + return vectorRemainder; + case OP_TYPE_GREATER_THAN: + return vectorGreater; + case OP_TYPE_GREATER_EQUAL: + return vectorGreaterEqual; + case OP_TYPE_LOWER_THAN: + return vectorLower; + case OP_TYPE_LOWER_EQUAL: + return vectorLowerEqual; + case OP_TYPE_EQUAL: + return vectorEqual; + case OP_TYPE_NOT_EQUAL: + return vectorNotEqual; + case OP_TYPE_IN: + return vectorIn; + case OP_TYPE_NOT_IN: + return vectorNotIn; + case OP_TYPE_LIKE: + return vectorLike; + case OP_TYPE_NOT_LIKE: + return vectorNotLike; + case OP_TYPE_MATCH: + return vectorMatch; + case OP_TYPE_NMATCH: + return vectorNotMatch; + case OP_TYPE_IS_NULL: + return vectorIsNull; + case OP_TYPE_IS_NOT_NULL: + return vectorNotNull; + case OP_TYPE_BIT_AND: + return vectorBitAnd; + case OP_TYPE_BIT_OR: + return vectorBitOr; + case OP_TYPE_IS_TRUE: + return vectorIsTrue; + default: + assert(0); + return NULL; + } +} + + diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt new file mode 100644 index 0000000000..caaf86264c --- /dev/null +++ b/source/libs/scalar/test/CMakeLists.txt @@ -0,0 +1,4 @@ +enable_testing() + +add_subdirectory(filter) +add_subdirectory(scalar) diff --git a/source/libs/scalar/test/filter/CMakeLists.txt b/source/libs/scalar/test/filter/CMakeLists.txt new file mode 100644 index 0000000000..b738285ded --- /dev/null +++ b/source/libs/scalar/test/filter/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build filter unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(filterTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + filterTest + PUBLIC os util common gtest qcom function nodes scalar +) + +TARGET_INCLUDE_DIRECTORIES( + filterTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp new file mode 100644 index 0000000000..db9df9ff08 --- /dev/null +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -0,0 +1,581 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" +#include "nodes.h" +#include "tlog.h" +#include "filter.h" + +namespace { + +int64_t flttLeftV = 21, flttRightV = 10; +double flttLeftVd = 21.0, flttRightVd = 10.0; + +void flttInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + + +void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)malloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); + SColumnRefNode *rnode = (SColumnRefNode *)node; + rnode->dataType.type = dataType; + rnode->dataType.bytes = dataBytes; + rnode->tupleId = 0; + + if (NULL == block) { + rnode->slotId = 2; + rnode->columnId = 55; + *pNode = (SNode *)rnode; + + return; + } + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = 0; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55; + idata.pData = (char *)value; + if (IS_VAR_DATA_TYPE(dataType)) { + idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); + for (int32_t i = 0; i < rowNum; ++i) { + idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + } + } + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = 2; + rnode->columnId = 55; + + *block = res; + } else { + SSDataBlock *res = *block; + + int32_t idx = taosArrayGetSize(res->pDataBlock); + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55 + idx; + idata.pData = (char *)value; + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = idx; + rnode->columnId = 55 + idx; + } + + *pNode = (SNode *)rnode; +} + +void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + +void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + + +void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + + +} + +TEST(timerangeTest, greater_and_lower) { + flttInitLogFile(); + + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + SNode *list[2] = {0}; + list[0] = opNode1; + list[1] = opNode2; + + flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + ASSERT_EQ(code, 0); + STimeWindow win = {0}; + code = filterGetTimeRange(filter, &win); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.ekey, tbig); +} + +#if 0 +TEST(columnTest, smallint_column_greater_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 2.5; + bool eRes[5] = {false, false, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, int_column_in_double_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + int32_t leftv[5] = {1, 2, 3, 4, 5}; + double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + SNodeList* list = nodesMakeList(); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3); + nodesListAppend(list, pRight); + flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_in_binary_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + char leftv[5][5]= {0}; + char rightv[3][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a' + i; + leftv[i][3] = 'b' + i; + leftv[i][4] = '0' + i; + varDataSetLen(leftv[i], 3); + } + for (int32_t i = 0; i < 2; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'b' + i; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + for (int32_t i = 2; i < 3; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'a' + i; + rightv[i][4] = 'a' + i; + varDataSetLen(rightv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SNodeList* list = nodesMakeList(); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]); + nodesListAppend(list, pRight); + flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); + flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_like_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + char rightv[64] = {0}; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, false, true, false, true}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a'; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + sprintf(&rightv[2], "%s", "__0"); + varDataSetLen(rightv, strlen(&rightv[2])); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + + +TEST(columnTest, binary_column_is_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, false, false, false, true}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_not_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, true, true, true, false}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(logicTest, and_or_and) { + +} + +TEST(logicTest, or_and_or) { + +} + + +TEST(opTest, smallint_column_greater_int_column) { + +} + +TEST(opTest, smallint_value_add_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + int16_t rightv[5]= {0, -5, -4, 23, 100}; + double eRes[5] = {1.0, -4, -3, 24, 101}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(opTest, bigint_column_multi_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int64_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + double eRes[5] = {0, 2, 6, 12, 20}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_and_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + int64_t eRes[5] = {0, 0, 2, 0, 4}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_or_float_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; + int64_t eRes[5] = {3, 3, 7, 5, 7}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_or_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 10.2; + int64_t eRes[5] = {11, 10, 11, 14, 15}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(opTest, binary_column_is_true) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, true, false, true, false}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} +#endif + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt new file mode 100644 index 0000000000..9dfbf3b5e8 --- /dev/null +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scalar unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + scalarTest + PUBLIC os util common gtest qcom function nodes scalar +) + +TARGET_INCLUDE_DIRECTORIES( + scalarTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp new file mode 100644 index 0000000000..2f0a60362d --- /dev/null +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -0,0 +1,1183 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" +#include "nodes.h" +#include "tlog.h" + +namespace { + +int64_t scltLeftV = 21, scltRightV = 10; +double scltLeftVd = 21.0, scltRightVd = 10.0; + +void scltInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + + +void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)malloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); + SColumnRefNode *rnode = (SColumnRefNode *)node; + rnode->dataType.type = dataType; + rnode->dataType.bytes = dataBytes; + rnode->tupleId = 0; + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = 0; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55; + idata.pData = (char *)value; + if (IS_VAR_DATA_TYPE(dataType)) { + idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); + for (int32_t i = 0; i < rowNum; ++i) { + idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + } + } + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = 2; + rnode->columnId = 55; + + *block = res; + } else { + SSDataBlock *res = *block; + + int32_t idx = taosArrayGetSize(res->pDataBlock); + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55 + idx; + idata.pData = (char *)value; + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = idx; + rnode->columnId = 55 + idx; + } + + *pNode = (SNode *)rnode; +} + +void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + + +void scltMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + + +void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + + +} + +TEST(constantTest, bigint_add_bigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BIGINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(v->datum.d, (scltLeftV + scltRightV)); +} + +TEST(constantTest, double_sub_bigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_DOUBLE, &scltLeftVd); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_SUB, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(v->datum.d, (scltLeftVd - scltRightV)); +} + +TEST(constantTest, tinyint_and_smallint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_TINYINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.i, (int64_t)scltLeftV & (int64_t)scltRightV); +} + +TEST(constantTest, bigint_or_double) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BIGINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &scltRightVd); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.i, (int64_t)scltLeftV | (int64_t)scltRightVd); +} + +TEST(constantTest, int_or_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char binaryStr[64] = {0}; + sprintf(&binaryStr[2], "%d", scltRightV); + varDataSetLen(binaryStr, strlen(&binaryStr[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, binaryStr); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); +} + + +TEST(constantTest, int_greater_double) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &scltRightVd); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); +} + +TEST(constantTest, int_greater_equal_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char binaryStr[64] = {0}; + sprintf(&binaryStr[2], "%d", scltRightV); + varDataSetLen(binaryStr, strlen(&binaryStr[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, binaryStr); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); +} + +TEST(constantTest, tinyint_lower_ubigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_TINYINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV < scltRightV); +} + +TEST(constantTest, usmallint_lower_equal_ubigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_USMALLINT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv <= rightv); +} + +TEST(constantTest, int_equal_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv == rightv); +} + +TEST(constantTest, int_equal_smallint2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 0, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv == rightv); +} + +TEST(constantTest, int_not_equal_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv != rightv); +} + +TEST(constantTest, int_not_equal_smallint2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 0, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv != rightv); +} + + + +TEST(constantTest, int_in_smallint1) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 1, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_in_smallint2) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 4, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_not_in_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 1, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_NOT_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_not_in_smallint2) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 4, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_NOT_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_like_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a_c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_like_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "ac"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_like_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a%c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_like_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "ac"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_match_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", ".*"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_match_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "abc.+"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_match_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a[1-9]c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NMATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_not_match_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a[ab]c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NMATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_is_null1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_is_null2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_is_not_null1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_is_not_null2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_add_int_is_true1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_add_int_is_true2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = -1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + + +TEST(constantTest, int_greater_int_is_true1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_greater_int_is_true2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 0; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, greater_and_lower) { + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v4); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + + + +TEST(columnTest, smallint_value_add_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + int16_t rightv[5]= {0, -5, -4, 23, 100}; + double eRes[5] = {1.0, -4, -3, 24, 101}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, bigint_column_multi_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int64_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + double eRes[5] = {0, 2, 6, 12, 20}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_and_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + int64_t eRes[5] = {0, 0, 2, 0, 4}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_or_float_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; + int64_t eRes[5] = {3, 3, 7, 5, 7}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_or_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 10.2; + int64_t eRes[5] = {11, 10, 11, 14, 15}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_greater_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 2.5; + bool eRes[5] = {false, false, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, int_column_in_double_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + int32_t leftv[5] = {1, 2, 3, 4, 5}; + double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_in_binary_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + char leftv[5][5]= {0}; + char rightv[3][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a' + i; + leftv[i][3] = 'b' + i; + leftv[i][4] = '0' + i; + varDataSetLen(leftv[i], 3); + } + for (int32_t i = 0; i < 2; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'b' + i; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + for (int32_t i = 2; i < 3; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'a' + i; + rightv[i][4] = 'a' + i; + varDataSetLen(rightv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_like_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + char rightv[64] = {0}; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, false, true, false, true}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a'; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + sprintf(&rightv[2], "%s", "__0"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_true) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, true, false, true, false}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, false, false, false, true}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_not_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, true, true, true, false}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, greater_and_lower) { + SNode *pcol1 = NULL, *pcol2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + SNode *list[2] = {0}; + int16_t v1[5]= {1, 2, 3, 4, 5}; + int32_t v2[5]= {5, 1, 4, 2, 6}; + int64_t v3[5]= {1, 2, 3, 4, 5}; + int32_t v4[5]= {5, 3, 4, 2, 6}; + bool eRes[5] = {false, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(v1)/sizeof(v1[0]); + scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); + scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); + scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); + scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculate(logicNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 784a864451..37ee5194c8 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -1,4 +1,3 @@ - aux_source_directory(src SYNC_SRC) add_library(sync ${SYNC_SRC}) diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h new file mode 100644 index 0000000000..9ca0de19c5 --- /dev/null +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_APPEND_ENTRIES_H +#define _TD_LIBS_SYNC_APPEND_ENTRIES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg); + +void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_APPEND_ENTRIES_H*/ diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h new file mode 100644 index 0000000000..8b5cbf1da5 --- /dev/null +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H +#define _TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H*/ diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h new file mode 100644 index 0000000000..34dfdb3d09 --- /dev/null +++ b/source/libs/sync/inc/syncElection.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_ELECTION_H +#define _TD_LIBS_SYNC_ELECTION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_ELECTION_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h new file mode 100644 index 0000000000..551ce83122 --- /dev/null +++ b/source/libs/sync/inc/syncInt.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_INT_H +#define _TD_LIBS_SYNC_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + +#define sFatal(...) \ + { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sError(...) \ + { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sWarn(...) \ + { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sInfo(...) \ + { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sDebug(...) \ + { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sTrace(...) \ + { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ + } \ + } + +typedef struct SSyncNode { + char path[TSDB_FILENAME_LEN]; + int8_t replica; + int8_t quorum; + int8_t selfIndex; + uint32_t vgId; + int32_t refCount; + int64_t rid; +} SSyncNode; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_INT_H*/ diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h new file mode 100644 index 0000000000..f410c8cf6e --- /dev/null +++ b/source/libs/sync/inc/syncMessage.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_MESSAGE_H +#define _TD_LIBS_SYNC_MESSAGE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncRaftEntry.h" +#include "taosdef.h" + +typedef enum ESyncMessageType { + SYNC_PING = 0, + SYNC_PING_REPLY, + SYNC_CLIENT_REQUEST, + SYNC_CLIENT_REQUEST_REPLY, + SYNC_REQUEST_VOTE, + SYNC_REQUEST_VOTE_REPLY, + SYNC_APPEND_ENTRIES, + SYNC_APPEND_ENTRIES_REPLY, +} ESyncMessageType; + +typedef struct SyncPing { + ESyncMessageType msgType; + const SSyncBuffer *pData; +} SyncPing; + +typedef struct SyncPingReply { + ESyncMessageType msgType; + const SSyncBuffer *pData; +} SyncPingReply; + +typedef struct SyncClientRequest { + ESyncMessageType msgType; + const SSyncBuffer *pData; + int64_t seqNum; + bool isWeak; +} SyncClientRequest; + +typedef struct SyncClientRequestReply { + ESyncMessageType msgType; + int32_t errCode; + const SSyncBuffer *pErrMsg; + const SSyncBuffer *pLeaderHint; +} SyncClientRequestReply; + +typedef struct SyncRequestVote { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + SyncGroupId vgId; + SyncIndex lastLogIndex; + SyncTerm lastLogTerm; +} SyncRequestVote; + +typedef struct SyncRequestVoteReply { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + SyncGroupId vgId; + bool voteGranted; +} SyncRequestVoteReply; + +typedef struct SyncAppendEntries { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + SyncIndex prevLogIndex; + SyncTerm prevLogTerm; + int32_t entryCount; + SSyncRaftEntry * logEntries; + SyncIndex commitIndex; +} SyncAppendEntries; + +typedef struct SyncAppendEntriesReply { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + bool success; + SyncIndex matchIndex; +} SyncAppendEntriesReply; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_MESSAGE_H*/ diff --git a/source/libs/sync/inc/syncOnMessage.h b/source/libs/sync/inc/syncOnMessage.h new file mode 100644 index 0000000000..07c44b6199 --- /dev/null +++ b/source/libs/sync/inc/syncOnMessage.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_ON_MESSAGE_H +#define _TD_LIBS_SYNC_ON_MESSAGE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncRaft.h" +#include "taosdef.h" + +void onMessage(SRaft *pRaft, void *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_ON_MESSAGE_H*/ diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h new file mode 100644 index 0000000000..0c7e573572 --- /dev/null +++ b/source/libs/sync/inc/syncRaft.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_H +#define _TD_LIBS_SYNC_RAFT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncMessage.h" +#include "taosdef.h" + +typedef struct SRaftId { + SyncNodeId nodeId; + SyncGroupId vgId; +} SRaftId; + +typedef struct SRaft { + SRaftId id; +} SRaft; + +int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); + +static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_H*/ diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h new file mode 100644 index 0000000000..adc82f2c5d --- /dev/null +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_TPL_H +#define _TD_LIBS_TPL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "taosdef.h" + +typedef struct SSyncRaftEntry { + SyncTerm term; + SyncIndex index; + SSyncBuffer data; + int8_t flag; +} SSyncRaftEntry; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_TPL_H*/ diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h new file mode 100644 index 0000000000..8c4b5116ea --- /dev/null +++ b/source/libs/sync/inc/syncRaftLog.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_LOG_H +#define _TD_LIBS_SYNC_RAFT_LOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "taosdef.h" + +int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); + +// get one log entry, user need to free pBuf->data +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); + +// update log store commit index with "index" +int32_t raftLogUpdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index); + +// truncate log with index, entries after the given index (>index) will be deleted +int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index); + +// return commit index of log +SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore); + +// return index of last entry +SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore); + +// return term of last entry +SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_LOG_H*/ diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h new file mode 100644 index 0000000000..4cb852f34a --- /dev/null +++ b/source/libs/sync/inc/syncRaftStore.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_STORE_H +#define _TD_LIBS_SYNC_RAFT_STORE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncRaft.h" +#include "taosdef.h" + +int32_t currentTerm(SyncTerm *pCurrentTerm); + +int32_t persistCurrentTerm(SyncTerm currentTerm); + +int32_t voteFor(SRaftId *pRaftId); + +int32_t persistVoteFor(SRaftId *pRaftId); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_STORE_H*/ diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h new file mode 100644 index 0000000000..40c5ff790b --- /dev/null +++ b/source/libs/sync/inc/syncReplication.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_REPLICATION_H +#define _TD_LIBS_SYNC_REPLICATION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_REPLICATION_H*/ diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h new file mode 100644 index 0000000000..3ff96bbe8f --- /dev/null +++ b/source/libs/sync/inc/syncRequestVote.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_REQUEST_VOTE_H +#define _TD_LIBS_SYNC_REQUEST_VOTE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg); + +void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_REQUEST_VOTE_H*/ diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h new file mode 100644 index 0000000000..033ac89bc2 --- /dev/null +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H +#define _TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H*/ diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h new file mode 100644 index 0000000000..3b6121578a --- /dev/null +++ b/source/libs/sync/inc/syncSnapshot.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_SNAPSHOT_H +#define _TD_LIBS_SYNC_SNAPSHOT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncRaft.h" +#include "taosdef.h" + +int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); + +int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_SNAPSHOT_H*/ diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h new file mode 100644 index 0000000000..8159d2566c --- /dev/null +++ b/source/libs/sync/inc/syncTimeout.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_TIMEOUT_H +#define _TD_LIBS_SYNC_TIMEOUT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void onTimeout(SRaft *pRaft, void *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_TIMEOUT_H*/ diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c deleted file mode 100644 index 7ded53b6e6..0000000000 --- a/source/libs/sync/src/sync.c +++ /dev/null @@ -1 +0,0 @@ -#include "sync.h" \ No newline at end of file diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c new file mode 100644 index 0000000000..1286108664 --- /dev/null +++ b/source/libs/sync/src/syncAppendEntries.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncAppendEntries.h" +#include "sync.h" + +void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { + +// TLA+ Spec +//AppendEntries(i, j) == +// /\ i /= j +// /\ state[i] = Leader +// /\ LET prevLogIndex == nextIndex[i][j] - 1 +// prevLogTerm == IF prevLogIndex > 0 THEN +// log[i][prevLogIndex].term +// ELSE +// 0 +// \* Send up to 1 entry, constrained by the end of the log. +// lastEntry == Min({Len(log[i]), nextIndex[i][j]}) +// entries == SubSeq(log[i], nextIndex[i][j], lastEntry) +// IN Send([mtype |-> AppendEntriesRequest, +// mterm |-> currentTerm[i], +// mprevLogIndex |-> prevLogIndex, +// mprevLogTerm |-> prevLogTerm, +// mentries |-> entries, +// \* mlog is used as a history variable for the proof. +// \* It would not exist in a real implementation. +// mlog |-> log[i], +// mcommitIndex |-> Min({commitIndex[i], lastEntry}), +// msource |-> i, +// mdest |-> j]) +// /\ UNCHANGED <> + +} + +void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { + +// TLA+ Spec +//HandleAppendEntriesRequest(i, j, m) == +// LET logOk == \/ m.mprevLogIndex = 0 +// \/ /\ m.mprevLogIndex > 0 +// /\ m.mprevLogIndex <= Len(log[i]) +// /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term +// IN /\ m.mterm <= currentTerm[i] +// /\ \/ /\ \* reject request +// \/ m.mterm < currentTerm[i] +// \/ /\ m.mterm = currentTerm[i] +// /\ state[i] = Follower +// /\ \lnot logOk +// /\ Reply([mtype |-> AppendEntriesResponse, +// mterm |-> currentTerm[i], +// msuccess |-> FALSE, +// mmatchIndex |-> 0, +// msource |-> i, +// mdest |-> j], +// m) +// /\ UNCHANGED <> +// \/ \* return to follower state +// /\ m.mterm = currentTerm[i] +// /\ state[i] = Candidate +// /\ state' = [state EXCEPT ![i] = Follower] +// /\ UNCHANGED <> +// \/ \* accept request +// /\ m.mterm = currentTerm[i] +// /\ state[i] = Follower +// /\ logOk +// /\ LET index == m.mprevLogIndex + 1 +// IN \/ \* already done with request +// /\ \/ m.mentries = << >> +// \/ /\ m.mentries /= << >> +// /\ Len(log[i]) >= index +// /\ log[i][index].term = m.mentries[1].term +// \* This could make our commitIndex decrease (for +// \* example if we process an old, duplicated request), +// \* but that doesn't really affect anything. +// /\ commitIndex' = [commitIndex EXCEPT ![i] = +// m.mcommitIndex] +// /\ Reply([mtype |-> AppendEntriesResponse, +// mterm |-> currentTerm[i], +// msuccess |-> TRUE, +// mmatchIndex |-> m.mprevLogIndex + +// Len(m.mentries), +// msource |-> i, +// mdest |-> j], +// m) +// /\ UNCHANGED <> +// \/ \* conflict: remove 1 entry +// /\ m.mentries /= << >> +// /\ Len(log[i]) >= index +// /\ log[i][index].term /= m.mentries[1].term +// /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |-> +// log[i][index2]] +// IN log' = [log EXCEPT ![i] = new] +// /\ UNCHANGED <> +// \/ \* no conflict: append entry +// /\ m.mentries /= << >> +// /\ Len(log[i]) = m.mprevLogIndex +// /\ log' = [log EXCEPT ![i] = +// Append(log[i], m.mentries[1])] +// /\ UNCHANGED <> +// /\ UNCHANGED <> +// + + +} diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c new file mode 100644 index 0000000000..4a9055e172 --- /dev/null +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncAppendEntriesReply.h" +#include "sync.h" + +void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { + +// TLA+ Spec +//HandleAppendEntriesResponse(i, j, m) == +// /\ m.mterm = currentTerm[i] +// /\ \/ /\ m.msuccess \* successful +// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] +// /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] +// \/ /\ \lnot m.msuccess \* not successful +// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = +// Max({nextIndex[i][j] - 1, 1})] +// /\ UNCHANGED <> +// /\ Discard(m) +// /\ UNCHANGED <> + +} diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncElection.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c new file mode 100644 index 0000000000..fbb969eb1c --- /dev/null +++ b/source/libs/sync/src/syncMain.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include "sync.h" +#include "syncInt.h" + +int32_t syncInit() { return 0; } + +void syncCleanUp() {} + +int64_t syncStart(const SSyncInfo* pSyncInfo) { return 0; } + +void syncStop(int64_t rid) {} + +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; } + +int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { return 0; } + +ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } + +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c new file mode 100644 index 0000000000..dcfc940f76 --- /dev/null +++ b/source/libs/sync/src/syncMessage.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncMessage.h" +#include "sync.h" +#include "syncRaft.h" + +void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncOnMessage.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c new file mode 100644 index 0000000000..85c2c6fe27 --- /dev/null +++ b/source/libs/sync/src/syncRaft.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaft.h" +#include "sync.h" + +int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } + +static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncRaftEntry.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c new file mode 100644 index 0000000000..4a5fc201b0 --- /dev/null +++ b/source/libs/sync/src/syncRaftLog.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaftLog.h" +#include "sync.h" + +int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } + +// get one log entry, user need to free pBuf->data +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf) { return 0; } + +// update log store commit index with "index" +int32_t raftLogupdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; } + +// truncate log with index, entries after the given index (>index) will be deleted +int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; } + +// return commit index of log +SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore) { return 0; } + +// return index of last entry +SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore) { return 0; } + +// return term of last entry +SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c new file mode 100644 index 0000000000..d45e53132c --- /dev/null +++ b/source/libs/sync/src/syncRaftStore.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaftStore.h" +#include "sync.h" + +int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } + +int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } + +int32_t voteFor(SRaftId *pRaftId) { return 0; } + +int32_t persistVoteFor(SRaftId *pRaftId) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncReplication.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c new file mode 100644 index 0000000000..c31ec0f34d --- /dev/null +++ b/source/libs/sync/src/syncRequestVote.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRequestVote.h" +#include "sync.h" + +void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { + +// TLA+ Spec +//RequestVote(i, j) == +// /\ state[i] = Candidate +// /\ j \notin votesResponded[i] +// /\ Send([mtype |-> RequestVoteRequest, +// mterm |-> currentTerm[i], +// mlastLogTerm |-> LastTerm(log[i]), +// mlastLogIndex |-> Len(log[i]), +// msource |-> i, +// mdest |-> j]) +// /\ UNCHANGED <> + +} + +void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { + +// TLA+ Spec +//HandleRequestVoteRequest(i, j, m) == +// LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) +// \/ /\ m.mlastLogTerm = LastTerm(log[i]) +// /\ m.mlastLogIndex >= Len(log[i]) +// grant == /\ m.mterm = currentTerm[i] +// /\ logOk +// /\ votedFor[i] \in {Nil, j} +// IN /\ m.mterm <= currentTerm[i] +// /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] +// \/ ~grant /\ UNCHANGED votedFor +// /\ Reply([mtype |-> RequestVoteResponse, +// mterm |-> currentTerm[i], +// mvoteGranted |-> grant, +// \* mlog is used just for the `elections' history variable for +// \* the proof. It would not exist in a real implementation. +// mlog |-> log[i], +// msource |-> i, +// mdest |-> j], +// m) +// /\ UNCHANGED <> + +} diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c new file mode 100644 index 0000000000..ba9787f00c --- /dev/null +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRequestVoteReply.h" +#include "sync.h" + +void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { + +// TLA+ Spec +//HandleRequestVoteResponse(i, j, m) == +// \* This tallies votes even when the current state is not Candidate, but +// \* they won't be looked at, so it doesn't matter. +// /\ m.mterm = currentTerm[i] +// /\ votesResponded' = [votesResponded EXCEPT ![i] = +// votesResponded[i] \cup {j}] +// /\ \/ /\ m.mvoteGranted +// /\ votesGranted' = [votesGranted EXCEPT ![i] = +// votesGranted[i] \cup {j}] +// /\ voterLog' = [voterLog EXCEPT ![i] = +// voterLog[i] @@ (j :> m.mlog)] +// \/ /\ ~m.mvoteGranted +// /\ UNCHANGED <> +// /\ Discard(m) +// /\ UNCHANGED <> + +} diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c new file mode 100644 index 0000000000..8a27f097d1 --- /dev/null +++ b/source/libs/sync/src/syncSnapshot.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncSnapshot.h" +#include "sync.h" +#include "syncRaft.h" + +int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } + +int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c new file mode 100644 index 0000000000..206dd70046 --- /dev/null +++ b/source/libs/sync/src/syncTimeout.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncTimeout.h" +#include "sync.h" + +void onTimeout(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp new file mode 100644 index 0000000000..47566d537e --- /dev/null +++ b/source/libs/sync/test/syncTest.cpp @@ -0,0 +1,7 @@ +#include + +int main() { + printf("test \n"); + return 0; +} + diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index d5a8cf5f84..d4d9bff56c 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -217,7 +217,7 @@ typedef struct SConnBuffer { char* buf; int len; int cap; - int left; + int total; } SConnBuffer; typedef void (*AsyncCB)(uv_async_t* handle); @@ -238,10 +238,11 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) void transDestroyAsyncPool(SAsyncPool* pool); int transSendAsync(SAsyncPool* pool, queue* mq); -int transInitBuffer(SConnBuffer* buf); -int transClearBuffer(SConnBuffer* buf); -int transDestroyBuffer(SConnBuffer* buf); -int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +int transInitBuffer(SConnBuffer* buf); +int transClearBuffer(SConnBuffer* buf); +int transDestroyBuffer(SConnBuffer* buf); +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +bool transReadComplete(SConnBuffer* connBuf); // int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 3c8c922d83..a36b671eb4 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -56,6 +56,7 @@ typedef struct { int8_t connType; int64_t index; char label[TSDB_LABEL_LEN]; + bool noPool; // pool or not char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 3bb7d103d7..72c1ff6893 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -64,6 +64,7 @@ typedef struct { void (*cfp)(void *parent, SRpcMsg *, SEpSet *); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); + bool noPool; int32_t refCount; void * parent; void * idPool; // handle to ID pool diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c3d3cfa2ab..48c15ca286 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -27,7 +27,7 @@ void* rpcOpen(const SRpcInit* pInit) { return NULL; } if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } pRpc->cfp = pInit->cfp; if (pInit->connType == TAOS_CONN_SERVER) { @@ -35,6 +35,8 @@ void* rpcOpen(const SRpcInit* pInit) { } else { pRpc->numOfThreads = pInit->numOfThreads; } + + pRpc->noPool = pInit->noPool; pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f1bd1ba980..8312c0217c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -84,8 +84,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// check whether already read complete packet from server -static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after read nbytes from socket @@ -126,6 +124,9 @@ static void clientHandleResp(SCliConn* conn) { pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); + // buf's mem alread translated to rpcMsg.pCont + transClearBuffer(&conn->readBuf); + SRpcMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = transContFromHead((char*)pHead); @@ -134,15 +135,15 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.ahandle = pCtx->ahandle; if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY) { + rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); } - tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), - inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), - ntohs(conn->locaddr.sin_port)); + tDebug("%s client conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pRpc->label, conn, + TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), rpcMsg.contLen); conn->secured = pHead->secured; if (conn->push != NULL && conn->ctnRdCnt != 0) { @@ -150,26 +151,26 @@ static void clientHandleResp(SCliConn* conn) { conn->push = NULL; } else { if (pCtx->pSem == NULL) { - tTrace("client conn %p handle resp", conn); + tTrace("%s client conn %p handle resp", pRpc->label, conn); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); } else { - tTrace("client conn(sync) %p handle resp", conn); + tTrace("%s client conn(sync) %p handle resp", pRpc->label, conn); memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg)); tsem_post(pCtx->pSem); } } conn->ctnRdCnt += 1; - // buf's mem alread translated to rpcMsg.pCont - transClearBuffer(&conn->readBuf); - uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb); SCliThrdObj* pThrd = conn->hostThrd; // user owns conn->persist = 1 if (conn->push == NULL && conn->persist == 0) { - addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + if (pRpc->noPool == true) { + } else { + addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + } } destroyCmsg(conn->data); conn->data = NULL; @@ -184,7 +185,6 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - tTrace("client conn %p start to destroy", pConn); SCliMsg* pMsg = pConn->data; tmsg_t msgType = TDMT_MND_CONNECT; @@ -213,6 +213,7 @@ static void clientHandleExcept(SCliConn* pConn) { } pConn->push = NULL; } + tTrace("%s client conn %p start to destroy", pCtx->pTransInst->label, pConn); if (pConn->push == NULL) { destroyCmsg(pConn->data); pConn->data = NULL; @@ -226,7 +227,7 @@ static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; - tTrace("client conn timeout, try to remove expire conn from conn pool"); + tTrace("%s, client conn timeout, try to remove expire conn from conn pool", pRpc->label); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); while (p != NULL) { @@ -306,23 +307,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); } -static bool clientReadComplete(SConnBuffer* data) { - STransMsgHead head; - int32_t headLen = sizeof(head); - if (data->len >= headLen) { - memcpy((char*)&head, data->buf, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); - if (msgLen > data->len) { - data->left = msgLen - data->len; - return false; - } else if (msgLen == data->len) { - data->left = 0; - return true; - } - } else { - return false; - } -} static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; @@ -337,8 +321,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { pBuf->len += nread; - if (clientReadComplete(pBuf)) { - uv_read_stop((uv_stream_t*)conn->stream); + if (transReadComplete(pBuf)) { tTrace("client conn %p read complete", conn); clientHandleResp(conn); } else { @@ -346,6 +329,10 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf } return; } + if (nread == UV_EOF) { + tError("client conn %p read error: %s", conn, uv_err_name(nread)); + clientHandleExcept(conn); + } assert(nread <= 0); if (nread == 0) { // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb @@ -353,7 +340,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf // read(2). return; } - if (nread < 0 || nread == UV_EOF) { + if (nread < 0) { tError("client conn %p read error: %s", conn, uv_err_name(nread)); clientHandleExcept(conn); } @@ -467,6 +454,7 @@ static void clientConnCb(uv_connect_t* req, int status) { static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("client work thread %p start to quit", pThrd); destroyCmsg(pMsg); + destroyConnPool(pThrd->pool); // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); uv_timer_stop(pThrd->timer); pThrd->quit = true; @@ -483,7 +471,10 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = NULL; if (pMsg->msg.handle == NULL) { - conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + if (pCtx->pTransInst->noPool == true) { + } else { + conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + } if (conn != NULL) { tTrace("client conn %p get from conn pool", conn); } @@ -512,7 +503,11 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; - + uv_tcp_nodelay((uv_tcp_t*)conn->stream, 1); + int ret = uv_tcp_keepalive((uv_tcp_t*)conn->stream, 1, 1); + if (ret) { + tTrace("client conn %p failed to set keepalive, %s", conn, uv_err_name(ret)); + } // write req handle conn->writeReq = malloc(sizeof(uv_write_t)); conn->writeReq->data = conn; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7aa5aa16f1..9a8607b0ed 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -205,6 +205,7 @@ int transInitBuffer(SConnBuffer* buf) { } int transClearBuffer(SConnBuffer* buf) { memset(buf, 0, sizeof(*buf)); + buf->total = -1; return 0; } int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { @@ -214,32 +215,38 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user * info--->| */ - static const int CAPACITY = 1024; + static const int CAPACITY = sizeof(STransMsgHead); SConnBuffer* p = connBuf; if (p->cap == 0) { p->buf = (char*)calloc(CAPACITY, sizeof(char)); p->len = 0; p->cap = CAPACITY; - p->left = -1; + p->total = -1; uvBuf->base = p->buf; uvBuf->len = CAPACITY; } else { - if (p->len >= p->cap) { - if (p->left == -1) { - p->cap *= 2; - p->buf = realloc(p->buf, p->cap); - } else if (p->len + p->left > p->cap) { - p->cap = p->len + p->left; - p->buf = realloc(p->buf, p->len + p->left); - } - } + p->cap = p->total; + p->buf = realloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; } return 0; } +// check whether already read complete +bool transReadComplete(SConnBuffer* connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy((char*)&head, connBuf->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + connBuf->total = msgLen; + } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} int transUnpackMsg(STransMsgHead* msgHead) {} diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 7ddeb99c9d..9fca371bf3 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -61,6 +61,7 @@ typedef struct SWorkThrdObj { SAsyncPool* asyncPool; // uv_async_t* workerAsync; // queue msg; + queue conn; pthread_mutex_t msgMtx; void* pTransInst; } SWorkThrdObj; @@ -95,6 +96,7 @@ static void uvOnAcceptCb(uv_stream_t* stream, int status); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); +static void uvShutDownCb(uv_shutdown_t* req, int status); static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); @@ -102,8 +104,7 @@ static void uvStartSendResp(SSrvMsg* msg); static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet -static bool readComplete(SConnBuffer* buf); -static SSrvConn* createConn(); +static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); static void uvDestroyConn(uv_handle_t* handle); @@ -117,51 +118,11 @@ static bool addHandleToWorkloop(void* arg); static bool addHandleToAcceptloop(void* arg); void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - /* - * formate of data buffer: - * |<--------------------------data from socket------------------------------->| - * |<------STransMsgHead------->|<-------------------other data--------------->| - */ SSrvConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; transAllocBuffer(pBuf, buf); } -// check data read from socket complete or not -// -static bool readComplete(SConnBuffer* data) { - // TODO(yihao): handle pipeline later - STransMsgHead head; - int32_t headLen = sizeof(head); - if (data->len >= headLen) { - memcpy((char*)&head, data->buf, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); - if (msgLen > data->len) { - data->left = msgLen - data->len; - return false; - } else if (msgLen == data->len) { - return true; - } else if (msgLen < data->len) { - return false; - // handle other packet later - } - } else { - return false; - } -} - -// static void uvDoProcess(SRecvInfo* pRecv) { -// // impl later -// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; -// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; -// SSrvConn* pConn = pRecv->thandle; -// tDump(pRecv->msg, pRecv->msgLen); -// terrno = 0; -// // SRpcReqContext* pContest; -// -// // do auth and check -//} - static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; @@ -241,7 +202,7 @@ static void uvHandleReq(SSrvConn* pConn) { } pConn->inType = pHead->msgType; - assert(transIsReq(pHead->msgType)); + // assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; pHead->code = htonl(pHead->code); @@ -266,9 +227,9 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->ref++; - tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), - ntohs(pConn->locaddr.sin_port)); + ntohs(pConn->locaddr.sin_port), rpcMsg.contLen); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -282,7 +243,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; tTrace("server conn %p read summary, total read: %d, current read: %d", conn, pBuf->len, (int)nread); - if (readComplete(pBuf)) { + if (transReadComplete(pBuf)) { tTrace("server conn %p alread read complete packet", conn); uvHandleReq(conn); } else { @@ -290,6 +251,14 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } + if (nread == UV_EOF) { + tError("server conn %p read error: %s", conn, uv_err_name(nread)); + if (conn->ref > 1) { + conn->ref++; // ref > 1 signed that write is in progress + } + destroyConn(conn, true); + return; + } if (nread == 0) { return; } @@ -302,8 +271,8 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - buf->base = malloc(sizeof(char)); buf->len = 2; + buf->base = calloc(1, sizeof(char) * buf->len); } void uvOnTimeoutCb(uv_timer_t* handle) { @@ -386,6 +355,7 @@ static void uvStartSendRespInternal(SSrvMsg* smsg) { static void uvStartSendResp(SSrvMsg* smsg) { // impl SSrvConn* pConn = smsg->pConn; + pConn->ref--; // if (taosArrayGetSize(pConn->srvMsgs) > 0) { tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); @@ -403,6 +373,16 @@ static void destroySmsg(SSrvMsg* smsg) { transFreeMsg(smsg->msg.pCont); free(smsg); } +static void destroyAllConn(SWorkThrdObj* pThrd) { + while (!QUEUE_IS_EMPTY(&pThrd->conn)) { + queue* h = QUEUE_HEAD(&pThrd->conn); + QUEUE_REMOVE(h); + QUEUE_INIT(h); + + SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue); + destroyConn(c, true); + } +} void uvWorkerAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; SWorkThrdObj* pThrd = item->pThrd; @@ -424,8 +404,11 @@ void uvWorkerAsyncCb(uv_async_t* handle) { continue; } if (msg->pConn == NULL) { - // free(msg); + + destroyAllConn(pThrd); + + uv_loop_close(pThrd->loop); uv_stop(pThrd->loop); } else { uvStartSendResp(msg); @@ -439,9 +422,16 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } static void uvAcceptAsyncCb(uv_async_t* async) { SServerObj* srv = async->data; + uv_close((uv_handle_t*)&srv->server, NULL); uv_stop(srv->loop); } +static void uvShutDownCb(uv_shutdown_t* req, int status) { + tDebug("conn failed to shut down: %s", uv_err_name(status)); + uv_close((uv_handle_t*)req->handle, uvDestroyConn); + free(req); +} + void uvOnAcceptCb(uv_stream_t* stream, int status) { if (status == -1) { return; @@ -491,7 +481,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SSrvConn* pConn = createConn(); + SSrvConn* pConn = createConn(pThrd); pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ @@ -507,6 +497,9 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; + // uv_tcp_nodelay(pConn->pTcp, 1); + // uv_tcp_keepalive(pConn->pTcp, 1, 1); + // init write request, just pConn->pWriter = calloc(1, sizeof(uv_write_t)); pConn->pWriter->data = pConn; @@ -560,6 +553,9 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); + // conn set + QUEUE_INIT(&pThrd->conn); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; @@ -598,8 +594,13 @@ void* workerThread(void* arg) { uv_run(pThrd->loop, UV_RUN_DEFAULT); } -static SSrvConn* createConn() { +static SSrvConn* createConn(void* hThrd) { + SWorkThrdObj* pThrd = hThrd; + SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + QUEUE_INIT(&pConn->queue); + + QUEUE_PUSH(&pThrd->conn, &pConn->queue); pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // tTrace("conn %p created", pConn); ++pConn->ref; @@ -610,7 +611,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { if (conn == NULL) { return; } - tTrace("server conn %p try to destroy", conn); + tTrace("server conn %p try to destroy, ref: %d", conn, conn->ref); if (--conn->ref > 0) { return; } @@ -621,20 +622,23 @@ static void destroyConn(SSrvConn* conn, bool clear) { destroySmsg(msg); } taosArrayDestroy(conn->srvMsgs); - - // destroySmsg(conn->pSrvMsg); - // conn->pSrvMsg = NULL; + QUEUE_REMOVE(&conn->queue); if (clear) { - uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); + tTrace("try to destroy conn %p", conn); + uv_tcp_close_reset(conn->pTcp, uvDestroyConn); + // uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); + // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); + // uv_unref((uv_handle_t*)conn->pTcp); + // uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); } } static void uvDestroyConn(uv_handle_t* handle) { SSrvConn* conn = handle->data; tDebug("server conn %p destroy", conn); uv_timer_stop(conn->pTimer); - free(conn->pTimer); - // free(conn->pTcp); + // free(conn->pTimer); + free(conn->pTcp); free(conn->pWriter); free(conn); } diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 6f80ea42ac..d1fefe2c72 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -16,69 +16,168 @@ #include #include #include "tep.h" +#include "tglobal.h" #include "trpc.h" +#include "ulog.h" using namespace std; -class TransObj { - public: - TransObj() { - const char *label = "APP"; - const char *secret = "secret"; - const char *user = "user"; - const char *ckey = "ckey"; +const char *label = "APP"; +const char *secret = "secret"; +const char *user = "user"; +const char *ckey = "ckey"; +class Server; +int port = 7000; +// server process +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +class Client { + public: + void Init(int nThread) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = (char *)label; - rpcInit.numOfThreads = 5; - rpcInit.cfp = NULL; - rpcInit.sessions = 100; - rpcInit.idleTime = 100; + rpcInit.numOfThreads = nThread; + rpcInit.cfp = processResp; rpcInit.user = (char *)user; rpcInit.secret = (char *)secret; rpcInit.ckey = (char *)ckey; rpcInit.spi = 1; - } - bool startCli() { - trans = NULL; + rpcInit.parent = this; rpcInit.connType = TAOS_CONN_CLIENT; - trans = rpcOpen(&rpcInit); - return trans != NULL ? true : false; + this->transCli = rpcOpen(&rpcInit); + tsem_init(&this->sem, 0, 0); } - bool startSrv() { - trans = NULL; - rpcInit.connType = TAOS_CONN_SERVER; - trans = rpcOpen(&rpcInit); - return trans != NULL ? true : false; + void SetResp(SRpcMsg *pMsg) { + // set up resp; + this->resp = *pMsg; + } + SRpcMsg *Resp() { return &this->resp; } + + void Restart() { + rpcClose(this->transCli); + this->transCli = rpcOpen(&rpcInit); } - bool sendAndRecv() { + void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { SEpSet epSet = {0}; epSet.inUse = 0; - addEpIntoEpSet(&epSet, "192.168.1.1", 7000); - addEpIntoEpSet(&epSet, "192.168.0.1", 7000); + addEpIntoEpSet(&epSet, "127.0.0.1", 7000); - if (trans == NULL) { - return false; - } - SRpcMsg rpcMsg = {0}, reqMsg = {0}; - reqMsg.pCont = rpcMallocCont(10); - reqMsg.contLen = 10; - reqMsg.ahandle = NULL; - rpcSendRecv(trans, &epSet, &reqMsg, &rpcMsg); - int code = rpcMsg.code; - std::cout << tstrerror(code) << std::endl; - return true; + rpcSendRequest(this->transCli, &epSet, req, NULL); + SemWait(); + *resp = this->resp; } - bool stop() { - rpcClose(trans); - trans = NULL; - return true; + void SemWait() { tsem_wait(&this->sem); } + void SemPost() { tsem_post(&this->sem); } + void Reset() {} + + ~Client() { + if (this->transCli) rpcClose(this->transCli); } private: - void * trans; + tsem_t sem; SRpcInit rpcInit; + void * transCli; + SRpcMsg resp; +}; +class Server { + public: + Server() { + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = port; + rpcInit.label = (char *)label; + rpcInit.numOfThreads = 5; + rpcInit.cfp = processReq; + rpcInit.user = (char *)user; + rpcInit.secret = (char *)secret; + rpcInit.ckey = (char *)ckey; + rpcInit.spi = 1; + rpcInit.connType = TAOS_CONN_SERVER; + } + void Start() { + this->transSrv = rpcOpen(&this->rpcInit); + taosMsleep(1000); + } + void Stop() { + if (this->transSrv == NULL) return; + rpcClose(this->transSrv); + this->transSrv = NULL; + } + void Restart() { + this->Stop(); + this->Start(); + } + ~Server() { + if (this->transSrv) rpcClose(this->transSrv); + this->transSrv = NULL; + } + + private: + SRpcInit rpcInit; + void * transSrv; +}; +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + Client *client = (Client *)parent; + client->SetResp(pMsg); + client->SemPost(); +} +class TransObj { + public: + TransObj() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + cqDebugFlag = 0; + tscEmbeddedInUtil = 1; + tsAsyncLog = 0; + + std::string path = "/tmp/transport"; + taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + } + cli = new Client; + cli->Init(1); + srv = new Server; + srv->Start(); + } + void RestartCli() { cli->Restart(); } + void StopSrv() { srv->Stop(); } + void RestartSrv() { srv->Restart(); } + void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); } + ~TransObj() { + delete cli; + delete srv; + } + + private: + Client *cli; + Server *srv; }; class TransEnv : public ::testing::Test { protected: @@ -93,11 +192,34 @@ class TransEnv : public ::testing::Test { TransObj *tr = NULL; }; -TEST_F(TransEnv, test_start_stop) { - assert(tr->startCli()); - assert(tr->sendAndRecv()); - assert(tr->stop()); - assert(tr->startSrv()); - assert(tr->stop()); +// TEST_F(TransEnv, 01sendAndRec) { +// for (int i = 0; i < 1; i++) { +// SRpcMsg req = {0}, resp = {0}; +// req.msgType = 0; +// req.pCont = rpcMallocCont(10); +// req.contLen = 10; +// tr->cliSendAndRecv(&req, &resp); +// assert(resp.code == 0); +// } +//} + +TEST_F(TransEnv, 02StopServer) { + for (int i = 0; i < 1; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } + SRpcMsg req = {0}, resp = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->StopSrv(); + // tr->RestartSrv(); + tr->cliSendAndRecv(&req, &resp); + + assert(resp.code != 0); } diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 7e542ef80f..bb526e0ba0 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -22,13 +22,12 @@ * windows implementation */ - -#include #include -#include +#include #include +#include -#pragma warning( disable : 4244 ) +#pragma warning(disable : 4244) typedef void (*win_timer_f)(int signo); @@ -40,8 +39,8 @@ void WINAPI taosWinOnTimer(UINT wTimerID, UINT msg, DWORD_PTR dwUser, DWORD_PTR } static MMRESULT timerId; -int taosInitTimer(win_timer_f callback, int ms) { - DWORD_PTR param = *((int64_t *) & callback); +int taosInitTimer(win_timer_f callback, int ms) { + DWORD_PTR param = *((int64_t *)&callback); timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); if (timerId == 0) { @@ -50,9 +49,7 @@ int taosInitTimer(win_timer_f callback, int ms) { return 0; } -void taosUninitTimer() { - timeKillEvent(timerId); -} +void taosUninitTimer() { timeKillEvent(timerId); } #elif defined(_TD_DARWIN_64) @@ -60,32 +57,32 @@ void taosUninitTimer() { * darwin implementation */ -#include #include +#include #include static void (*timer_callback)(int); -static int timer_ms = 0; -static pthread_t timer_thread; -static int timer_kq = -1; -static volatile int timer_stop = 0; +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; -static void* timer_routine(void *arg) { +static void* timer_routine(void* arg) { (void)arg; setThreadName("timer"); - int r = 0; + int r = 0; struct timespec to = {0}; - to.tv_sec = timer_ms / 1000; - to.tv_nsec = (timer_ms % 1000) * 1000000; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; while (!timer_stop) { struct kevent64_s kev[10] = {0}; - r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); - if (r!=0) { + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev) / sizeof(kev[0]), 0, &to); + if (r != 0) { fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); abort(); } - timer_callback(SIGALRM); // just mock + timer_callback(SIGALRM); // just mock } return NULL; @@ -93,11 +90,13 @@ static void* timer_routine(void *arg) { int taosInitTimer(void (*callback)(int), int ms) { int r = 0; - timer_ms = ms; + timer_kq = -1; + timer_stop = 0; + timer_ms = ms; timer_callback = callback; timer_kq = kqueue(); - if (timer_kq==-1) { + if (timer_kq == -1) { fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); @@ -144,10 +143,10 @@ static void taosDeleteTimer(void *tharg) { timer_delete(*pTimer); } -static pthread_t timerThread; -static timer_t timerId; +static pthread_t timerThread; +static timer_t timerId; static volatile bool stopTimer = false; -static void *taosProcessAlarmSignal(void *tharg) { +static void * taosProcessAlarmSignal(void *tharg) { // Block the signal sigset_t sigset; sigemptyset(&sigset); @@ -159,18 +158,18 @@ static void *taosProcessAlarmSignal(void *tharg) { setThreadName("tmr"); - #ifdef _ALPINE - sevent.sigev_notify = SIGEV_THREAD; - sevent.sigev_value.sival_int = syscall(__NR_gettid); - #else - sevent.sigev_notify = SIGEV_THREAD_ID; - sevent._sigev_un._tid = syscall(__NR_gettid); - #endif - +#ifdef _ALPINE + sevent.sigev_notify = SIGEV_THREAD; + sevent.sigev_value.sival_int = syscall(__NR_gettid); +#else + sevent.sigev_notify = SIGEV_THREAD_ID; + sevent._sigev_un._tid = syscall(__NR_gettid); +#endif + sevent.sigev_signo = SIGALRM; if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) { - //printf("Failed to create timer"); + // printf("Failed to create timer"); } pthread_cleanup_push(taosDeleteTimer, &timerId); @@ -182,36 +181,37 @@ static void *taosProcessAlarmSignal(void *tharg) { ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; if (timer_settime(timerId, 0, &ts, NULL)) { - //printf("Failed to init timer"); + // printf("Failed to init timer"); return NULL; } int signo; while (!stopTimer) { if (sigwait(&sigset, &signo)) { - //printf("Failed to wait signal: number %d", signo); + // printf("Failed to wait signal: number %d", signo); continue; } /* //printf("Signal handling: number %d ......\n", signo); */ callback(0); } - + pthread_cleanup_pop(1); return NULL; } int taosInitTimer(void (*callback)(int), int ms) { + stopTimer = false; pthread_attr_t tattr; pthread_attr_init(&tattr); int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); pthread_attr_destroy(&tattr); if (code != 0) { - //printf("failed to create timer thread"); + // printf("failed to create timer thread"); return -1; } else { - //printf("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); + // printf("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); } return 0; @@ -220,7 +220,7 @@ int taosInitTimer(void (*callback)(int), int ms) { void taosUninitTimer() { stopTimer = true; - //printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); + // printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); pthread_join(timerThread, NULL); } diff --git a/source/util/src/compare.c b/source/util/src/compare.c index a1c1625c34..1b1fa41754 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -23,23 +23,51 @@ #include "thash.h" #include "types.h" #include "ulog.h" +#include "tdef.h" +#include "taos.h" -int32_t setCompareBytes1(const void *pLeft, const void *pRight) { +int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; } -int32_t setCompareBytes2(const void *pLeft, const void *pRight) { +int32_t setChkInBytes2(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; } -int32_t setCompareBytes4(const void *pLeft, const void *pRight) { +int32_t setChkInBytes4(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; } -int32_t setCompareBytes8(const void *pLeft, const void *pRight) { +int32_t setChkInBytes8(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; } +int32_t setChkNotInBytes1(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; +} + +int32_t setChkNotInBytes2(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; +} + +int32_t setChkNotInBytes4(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; +} + +int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; +} + + +int32_t compareChkInString(const void *pLeft, const void* pRight) { + return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} + +int32_t compareChkNotInString(const void *pLeft, const void* pRight) { + return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} + + int32_t compareInt8Val(const void *pLeft, const void *pRight) { int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); if (left > right) return 1; @@ -393,6 +421,156 @@ int32_t taosArrayCompareString(const void* a, const void* b) { return compareLenPrefixedStr(x, y); } -int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { - return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; + +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); + char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); + free(pattern); + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } + +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); + + wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + + int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); + free(pattern); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + + +__compar_fn_t getComparFunc(int32_t type, int32_t optr) { + __compar_fn_t comparFn = NULL; + + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkInBytes8; + default: + assert(0); + } + } + + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkNotInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkNotInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkNotInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkNotInBytes8; + default: + assert(0); + } + } + + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; + case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_BINARY: { + if (optr == OP_TYPE_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == OP_TYPE_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternMatch; + } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternNotMatch; + } else if (optr == OP_TYPE_IN) { + comparFn = compareChkInString; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = compareChkNotInString; + } else { /* normal relational comparFn */ + comparFn = compareLenPrefixedStr; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + if (optr == OP_TYPE_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == OP_TYPE_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == OP_TYPE_LIKE) { + comparFn = compareWStrPatternMatch; + } else if (optr == OP_TYPE_NOT_LIKE) { + comparFn = compareWStrPatternNotMatch; + } else if (optr == OP_TYPE_IN) { + comparFn = compareChkInString; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = compareChkNotInString; + } else { + comparFn = compareLenPrefixedWStr; + } + break; + } + + case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; + case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} + + diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index e91b11026b..2d7f8a5b03 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -164,20 +164,20 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { return fn; } -//int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { -// return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); -//} -// -//int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { -// return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); -//} +int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); +} + +int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); +} _equal_fn_t taosGetDefaultEqualFunction(int32_t type) { _equal_fn_t fn = NULL; -// switch (type) { -// case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; -// case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; -// default: fn = memcmp; break; -// } + switch (type) { + case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; + case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; + default: fn = memcmp; break; + } return fn; } diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 556e8f8060..13367843fc 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -32,6 +32,10 @@ int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t return tjsonAddStringToObject(pJson, pName, tmp); } +int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) { + return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); +} + int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } @@ -74,3 +78,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } + +char* tjsonToUnformattedString(const SJson* pJson) { + return cJSON_PrintUnformatted((cJSON*)pJson); +} \ No newline at end of file diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index b989a3f56c..ca0c6e9a24 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -13,19 +13,49 @@ * along with this program. If not, see . */ +#include "ttimer.h" #include "os.h" +#include "taoserror.h" #include "tlog.h" #include "tsched.h" -#include "ttimer.h" #include "tutil.h" -#include "taoserror.h" -#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrFatal(...) \ + { \ + if (tmrDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrError(...) \ + { \ + if (tmrDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrWarn(...) \ + { \ + if (tmrDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrInfo(...) \ + { \ + if (tmrDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrDebug(...) \ + { \ + if (tmrDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrTrace(...) \ + { \ + if (tmrDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } #define TIMER_STATE_WAITING 0 #define TIMER_STATE_EXPIRED 1 @@ -90,7 +120,7 @@ static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; static int numOfTmrCtrl = 0; -int taosTmrThreads = 1; +int taosTmrThreads = 1; static uintptr_t nextTimerId = 0; static time_wheel_t wheels[] = { @@ -118,7 +148,7 @@ static void timerDecRef(tmr_obj_t* timer) { static void lockTimerList(timer_list_t* list) { int64_t tid = taosGetSelfPthreadId(); - int i = 0; + int i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { sched_yield(); @@ -275,11 +305,11 @@ static void addToExpired(tmr_obj_t* head) { const char* fmt = "%s adding expired timer[id=%" PRIuPTR ", fp=%p, param=%p] to queue."; while (head != NULL) { - uintptr_t id = head->id; + uintptr_t id = head->id; tmr_obj_t* next = head->next; tmrDebug(fmt, head->ctrl->label, id, head->fp, head->param); - SSchedMsg schedMsg; + SSchedMsg schedMsg; schedMsg.fp = NULL; schedMsg.tfp = processExpiredTimer; schedMsg.msg = NULL; @@ -490,6 +520,8 @@ static void taosTmrModuleInit(void) { return; } + memset(&timerMap, 0, sizeof(timerMap)); + for (uint32_t i = 0; i < tsMaxTmrCtrl - 1; ++i) { tmr_ctrl_t* ctrl = tmrCtrls + i; ctrl->next = ctrl + 1; @@ -569,7 +601,8 @@ void taosTmrCleanUp(void* handle) { unusedTmrCtrl = ctrl; pthread_mutex_unlock(&tmrCtrlMutex); - if (numOfTmrCtrl <=0) { + tmrDebug("time controller's tmr ctrl size: %d", numOfTmrCtrl); + if (numOfTmrCtrl <= 0) { taosUninitTimer(); taosCleanUpScheduler(tmrQhandle); @@ -584,7 +617,7 @@ void taosTmrCleanUp(void* handle) { for (size_t i = 0; i < timerMap.size; i++) { timer_list_t* list = timerMap.slots + i; - tmr_obj_t* t = list->timers; + tmr_obj_t* t = list->timers; while (t != NULL) { tmr_obj_t* next = t->mnext; free(t); @@ -594,6 +627,8 @@ void taosTmrCleanUp(void* handle) { free(timerMap.slots); free(tmrCtrls); - tmrDebug("timer module is cleaned up"); + tmrCtrls = NULL; + unusedTmrCtrl = NULL; + tmrModuleInit = PTHREAD_ONCE_INIT; // to support restart } }