diff --git a/cmake/cmake.options b/cmake/cmake.options index e84d02800c..e19c10f6b2 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -47,13 +47,13 @@ option( option( BUILD_WITH_UV "If build with libuv" - OFF + ON ) option( BUILD_WITH_UV_TRANS "If build with libuv_trans " - OFF + ON ) option( diff --git a/include/common/tcompare.h b/include/common/tcompare.h index 8476a79e92..5dcbf37b90 100644 --- a/include/common/tcompare.h +++ b/include/common/tcompare.h @@ -23,8 +23,12 @@ extern "C" { #endif -int32_t compareStrPatternComp(const void* pLeft, const void* pRight); -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight); +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight); + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight); +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight); + __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); @@ -33,4 +37,4 @@ int32_t doCompare(const char* a, const char* b, int32_t type, size_t size) } #endif -#endif /*_TD_TCOMPARE_H_*/ \ No newline at end of file +#endif /*_TD_TCOMPARE_H_*/ diff --git a/include/common/tname.h b/include/common/tname.h index 12a0d34cb4..47028fbce1 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,6 +16,11 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H +#ifdef __cplusplus +extern "C" { +#endif + + #include "tdef.h" #include "tmsg.h" @@ -59,4 +64,9 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId); SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +#ifdef __cplusplus +} +#endif + + #endif // TDENGINE_TNAME_H diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 505685f8d5..5a1e442277 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -78,9 +78,12 @@ typedef struct { case TSDB_DATA_TYPE_UINT: \ (_v) = (_finalType)GET_UINT32_VAL(_data); \ break; \ - default: \ + case TSDB_DATA_TYPE_INT: \ (_v) = (_finalType)GET_INT32_VAL(_data); \ break; \ + default: \ + (_v) = (_finalType)varDataLen(_data); \ + break; \ } \ } while (0) @@ -115,9 +118,11 @@ typedef struct { case TSDB_DATA_TYPE_UINT: \ *(uint32_t *)(_v) = (uint32_t)(_data); \ break; \ - default: \ + case TSDB_DATA_TYPE_INT: \ *(int32_t *)(_v) = (int32_t)(_data); \ break; \ + default: \ + break; \ } \ } while (0) @@ -138,6 +143,9 @@ typedef struct { #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) +#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) +#define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) + static FORCE_INLINE bool isNull(const void *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -205,6 +213,7 @@ void* getDataMax(int32_t type); #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) +#define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL) #ifdef __cplusplus } diff --git a/include/libs/function/function.h b/include/libs/function/function.h index aef5f7fec4..bb5d51686d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -226,13 +226,18 @@ typedef struct SAggFunctionInfo { int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; -struct SScalarFuncParam; +typedef struct SScalarParam { + void* data; + int32_t num; + int32_t type; + int32_t bytes; +} SScalarParam; typedef struct SScalarFunctionInfo { char name[FUNCTIONS_NAME_MAX_LENGTH]; int8_t type; // scalar function or aggregation function uint32_t functionId; // index of scalar function - void (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput); + void (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput); } SScalarFunctionInfo; typedef struct SMultiFunctionsDesc { @@ -285,10 +290,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); -struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num); -void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num); -struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index); - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // fill api struct SFillInfo; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 69673804fe..e77ccaa225 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "querynodes.h" +#include "function.h" typedef enum EFunctionType { // aggregate function @@ -117,6 +118,13 @@ typedef struct SFuncExecFuncs { FExecFinalize finalize; } SFuncExecFuncs; +typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +typedef struct SScalarFuncExecFuncs { + FScalarExecProcess process; +} SScalarFuncExecFuncs; + + int32_t fmFuncMgtInit(); int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); @@ -134,6 +142,7 @@ bool fmIsTimeorderFunc(int32_t funcId); int32_t fmFuncScanType(int32_t funcId); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); +int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); #ifdef __cplusplus } diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6a0422cd80..3ef832a69b 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -132,7 +132,7 @@ bool nodesEqualNode(const SNode* a, const SNode* b); SNode* nodesCloneNode(const SNode* pNode); SNodeList* nodesCloneList(const SNodeList* pList); -int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); +int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); #ifdef __cplusplus diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 024967289a..7c75562759 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -306,9 +306,10 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); +void *nodesGetValueFromNode(SValueNode *pNode); #ifdef __cplusplus } #endif -#endif /*_TD_QUERY_NODES_H_*/ \ No newline at end of file +#endif /*_TD_QUERY_NODES_H_*/ diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h new file mode 100644 index 0000000000..8db74f4587 --- /dev/null +++ b/include/libs/scalar/filter.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_FILTER_H +#define TDENGINE_FILTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SFilterInfo SFilterInfo; + +typedef struct SFilterColumnParam{ + int32_t numOfCols; + SArray* pDataBlock; +} SFilterColumnParam; + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_FILTER_H \ No newline at end of file diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h new file mode 100644 index 0000000000..4dbdb2e8bb --- /dev/null +++ b/include/libs/scalar/scalar.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_SCALAR_H +#define TDENGINE_SCALAR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "function.h" +#include "nodes.h" +#include "querynodes.h" + +typedef struct SFilterInfo SFilterInfo; + + +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); +int32_t scalarGetOperatorParamNum(EOperatorType type); + +int32_t vectorGetConvertType(int32_t type1, int32_t type2); +int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SCALAR_H diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 00ba1120e7..0ec741ec3e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -137,22 +137,21 @@ typedef struct { } SSyncInfo; -// will be defined in syncInt.h, here just for complie -typedef struct SSyncNode { -} SSyncNode; +struct SSyncNode; +typedef struct SSyncNode SSyncNode; int32_t syncInit(); void syncCleanUp(); -int64_t syncStart(const SSyncInfo*); +int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg*); +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); // int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); ESyncState syncGetMyRole(int64_t rid); -void syncGetNodesRole(int64_t rid, SNodesRole*); +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); extern int32_t sDebugFlag; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index f913ba06d0..538aeb1a0e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -64,6 +64,7 @@ typedef struct SRpcInit { int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int idleTime; // milliseconds, 0 means idle timer is disabled + bool noPool; // create conn pool or not // the following is for client app ecurity only char *user; // user name char spi; // security parameter index diff --git a/include/util/compare.h b/include/util/compare.h index 70a8134b35..6305334e02 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -49,10 +49,19 @@ int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, int32_t taosArrayCompareString(const void *a, const void *b); -int32_t setCompareBytes1(const void *pLeft, const void *pRight); -int32_t setCompareBytes2(const void *pLeft, const void *pRight); -int32_t setCompareBytes4(const void *pLeft, const void *pRight); -int32_t setCompareBytes8(const void *pLeft, const void *pRight); +int32_t setChkInBytes1(const void *pLeft, const void *pRight); +int32_t setChkInBytes2(const void *pLeft, const void *pRight); +int32_t setChkInBytes4(const void *pLeft, const void *pRight); +int32_t setChkInBytes8(const void *pLeft, const void *pRight); + +int32_t setChkNotInBytes1(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes2(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes4(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes8(const void *pLeft, const void *pRight); + +int32_t compareChkInString(const void *pLeft, const void *pRight); +int32_t compareChkNotInString(const void *pLeft, const void *pRight); + int32_t compareInt8Val(const void *pLeft, const void *pRight); int32_t compareInt16Val(const void *pLeft, const void *pRight); @@ -74,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight); int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); -int32_t compareFindItemInSet(const void *pLeft, const void *pRight); int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); @@ -92,6 +100,9 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); +__compar_fn_t getComparFunc(int32_t type, int32_t optr); + + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index f57d9b6f35..a4c333a4d2 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -118,16 +118,18 @@ do { \ #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_ISNULL 8 -#define TSDB_RELATION_NOTNULL 9 -#define TSDB_RELATION_IN 10 +#define TSDB_RELATION_NOT_LIKE 8 +#define TSDB_RELATION_ISNULL 9 +#define TSDB_RELATION_NOTNULL 10 +#define TSDB_RELATION_IN 11 +#define TSDB_RELATION_NOT_IN 12 -#define TSDB_RELATION_AND 11 -#define TSDB_RELATION_OR 12 -#define TSDB_RELATION_NOT 13 +#define TSDB_RELATION_AND 13 +#define TSDB_RELATION_OR 14 +#define TSDB_RELATION_NOT 15 -#define TSDB_RELATION_MATCH 14 -#define TSDB_RELATION_NMATCH 15 +#define TSDB_RELATION_MATCH 16 +#define TSDB_RELATION_NMATCH 17 #define TSDB_BINARY_OP_ADD 4000 #define TSDB_BINARY_OP_SUBTRACT 4001 diff --git a/include/util/thash.h b/include/util/thash.h index 3a614a73a6..1f72045a83 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -55,6 +55,8 @@ uint32_t taosIntHash_64(const char *key, uint32_t len); _hash_fn_t taosGetDefaultHashFunction(int32_t type); +_equal_fn_t taosGetDefaultEqualFunction(int32_t type); + typedef struct SHashNode { struct SHashNode *next; uint32_t hashVal; // the hash value of key @@ -258,6 +260,8 @@ void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashRelease(SHashObj *pHashObj, void *p); +void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); + #ifdef __cplusplus } diff --git a/include/util/tjson.h b/include/util/tjson.h index a4eb6e5385..a0c2fef05b 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -30,6 +30,7 @@ void tjsonDelete(SJson* pJson); SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number); +int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number); int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal); int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); @@ -42,6 +43,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj); typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj); char* tjsonToString(const SJson* pJson); +char* tjsonToUnformattedString(const SJson* pJson); #ifdef __cplusplus } diff --git a/include/util/types.h b/include/util/types.h index cb25448cc4..25688df56c 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -66,6 +66,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { // #else #define GET_FLOAT_VAL(x) (*(float *)(x)) #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_BIGINT_VAL(x, y) { (*(int64_t *)(x)) = (int64_t)(y); } #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c index 3c0e6df735..ef441c97c7 100644 --- a/source/common/src/tcompare.c +++ b/source/common/src/tcompare.c @@ -15,116 +15,6 @@ #include "tcompare.h" -int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); - memcpy(buf, varDataVal(pLeft), sz); - buf[sz] = 0; - - int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - free(buf); - free(pattern); - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - - wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); - free(pattern); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -__compar_fn_t getComparFunc(int32_t type, int32_t optr) { - __compar_fn_t comparFn = NULL; - - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - return setCompareBytes1; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - return setCompareBytes2; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - return setCompareBytes4; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - return setCompareBytes8; - default: - assert(0); - } - } - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; - case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; - case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternComp; - } else if (optr == TSDB_RELATION_IN) { - comparFn = compareFindItemInSet; - } else { /* normal relational comparFn */ - comparFn = compareLenPrefixedStr; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { - comparFn = compareWStrPatternComp; - } else if (optr == TSDB_RELATION_IN) { - comparFn = compareFindItemInSet; - } else { - comparFn = compareLenPrefixedWStr; - } - break; - } - - case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; - case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; - - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { __compar_fn_t comparFn = NULL; @@ -218,4 +108,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { } } } -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index a0ba71a1eb..4e35baf905 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -155,7 +155,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; + SDnode * pDnode = parent; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pRsp->msgType; @@ -193,6 +193,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; rpcInit.parent = pDnode; + rpcInit.noPool = true; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); @@ -218,7 +219,7 @@ static void dndCleanupClient(SDnode *pDnode) { } static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; + SDnode * pDnode = param; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pReq->msgType; @@ -312,7 +313,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/impl/test/sut/inc/client.h index 9cf688fc02..925680d528 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/client.h +++ b/source/dnode/mgmt/impl/test/sut/inc/client.h @@ -21,16 +21,21 @@ class TestClient { bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port); void Cleanup(); + void DoInit(); + SRpcMsg* SendReq(SRpcMsg* pReq); void SetRpcRsp(SRpcMsg* pRsp); tsem_t* GetSem(); + void Restart(); private: char fqdn[TSDB_FQDN_LEN]; uint16_t port; + char user[128]; + char pass[128]; void* clientRpc; SRpcMsg* pRsp; tsem_t sem; }; -#endif /* _TD_TEST_CLIENT_H_ */ \ No newline at end of file +#endif /* _TD_TEST_CLIENT_H_ */ diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 23913b0531..250d563a8b 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -20,10 +20,10 @@ #include "os.h" #include "dnode.h" -#include "tmsg.h" #include "tconfig.h" #include "tdataformat.h" #include "tglobal.h" +#include "tmsg.h" #include "tnote.h" #include "trpc.h" #include "tthread.h" @@ -39,6 +39,7 @@ class Testbase { void Restart(); void ServerStop(); void ServerStart(); + void ClientRestart(); SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen); private: @@ -100,7 +101,7 @@ class Testbase { { \ char* bytes = (char*)calloc(1, len); \ for (int32_t i = 0; i < len - 1; ++i) { \ - bytes[i] = b; \ + bytes[i] = b; \ } \ EXPECT_STREQ(test.GetShowBinary(len), bytes); \ } @@ -138,4 +139,4 @@ class Testbase { #define IgnoreTimestamp() \ { test.GetShowTimestamp(); } -#endif /* _TD_TEST_BASE_H_ */ \ No newline at end of file +#endif /* _TD_TEST_BASE_H_ */ diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index 8403dbf034..b89cb02834 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -13,33 +13,41 @@ * along with this program. If not, see . */ -#include "tep.h" #include "sut.h" +#include "tep.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; client->SetRpcRsp(pRsp); - uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pRsp->msgType), pRsp->code); + uInfo("x response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen); tsem_post(client->GetSem()); } -void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; +void TestClient::SetRpcRsp(SRpcMsg* rsp) { + if (this->pRsp) { + free(this->pRsp); + } + this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg)); + this->pRsp->msgType = rsp->msgType; + this->pRsp->code = rsp->code; + this->pRsp->pCont = rsp->pCont; + this->pRsp->contLen = rsp->contLen; +}; tsem_t* TestClient::GetSem() { return &sem; } -bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { +void TestClient::DoInit() { char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); - SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = (char*)"DND-C"; + rpcInit.label = (char*)"shell"; rpcInit.numOfThreads = 1; rpcInit.cfp = processClientRsp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 30 * 1000; - rpcInit.user = (char*)user; + rpcInit.user = (char*)this->user; rpcInit.ckey = (char*)"key"; rpcInit.parent = this; rpcInit.secret = (char*)secretEncrypt; @@ -47,11 +55,16 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint clientRpc = rpcOpen(&rpcInit); ASSERT(clientRpc); + tsem_init(&this->sem, 0, 0); +} - tsem_init(&sem, 0, 0); +bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { strcpy(this->fqdn, fqdn); + strcpy(this->user, user); + strcpy(this->pass, pass); this->port = port; - + this->pRsp = NULL; + this->DoInit(); return true; } @@ -60,11 +73,16 @@ void TestClient::Cleanup() { rpcClose(clientRpc); } +void TestClient::Restart() { + this->Cleanup(); + this->DoInit(); +} SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) { SEpSet epSet = {0}; addEpIntoEpSet(&epSet, fqdn, port); rpcSendRequest(clientRpc, &epSet, pReq, NULL); tsem_wait(&sem); + uInfo("y response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen); return pRsp; } diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 09a738be3b..771c5886ef 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -21,9 +21,9 @@ void Testbase::InitLog(const char* path) { mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; - tmrDebugFlag = 0; - uDebugFlag = 0; - rpcDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; qDebugFlag = 0; wDebugFlag = 0; sDebugFlag = 0; @@ -66,16 +66,21 @@ void Testbase::Init(const char* path, int16_t port) { void Testbase::Cleanup() { tFreeSTableMetaRsp(&metaRsp); - server.Stop(); client.Cleanup(); + taosMsleep(10); + server.Stop(); dndCleanup(); } -void Testbase::Restart() { server.Restart(); } +void Testbase::Restart() { + server.Restart(); + client.Restart(); +} void Testbase::ServerStop() { server.Stop(); } void Testbase::ServerStart() { server.DoStart(); } +void Testbase::ClientRestart() { client.Restart(); } SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { SRpcMsg rpcMsg = {0}; @@ -194,4 +199,4 @@ int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; } STableMetaRsp* Testbase::GetShowMeta() { return &metaRsp; } -SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } \ No newline at end of file +SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 699ccab92c..64b4aa6dd7 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -77,7 +77,7 @@ static void mndTransReExecute(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } @@ -89,7 +89,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } @@ -404,7 +404,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { + if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && + pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); @@ -439,7 +440,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; tmsg_t msgType = pMsg->rpcMsg.msgType; - void *ahandle = pMsg->rpcMsg.ahandle; + void * ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType & 1U); mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index d4e308268a..b8a0e61ca3 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -190,6 +190,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); server2.Stop(); + taosMsleep(1000); + // test.ClientRestart(); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); @@ -226,6 +229,7 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { { // server start, wait until the rollback finished server2.DoStart(); + test.ClientRestart(); taosMsleep(1000); int32_t retry = 0; @@ -248,7 +252,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { ASSERT_NE(retry, retryMax); } } - TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started @@ -315,4 +318,4 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { ASSERT_NE(retry, retryMax); } -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index d4c40dd428..88d4fc4f75 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -46,8 +46,10 @@ class MndTestTrans : public ::testing::Test { free(buffer); taosFsyncFile(fd); taosCloseFile(fd); + taosMsleep(1000); test.ServerStart(); + test.ClientRestart(); } static Testbase test; @@ -284,6 +286,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { } } + // create db // partial create stb // drop db failed diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index d8ce599be1..61b99beeb7 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -617,6 +617,7 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { // restart test.Restart(); + taosMsleep(1000); test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); @@ -631,4 +632,4 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { CheckTimestamp(); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f779949f14..4b47413715 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -126,7 +126,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - metaRsp.dbId = htobe64(pVnode->config.dbId); + metaRsp.dbId = pVnode->config.dbId; memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); strcpy(metaRsp.tbName, infoReq.tbName); if (pTbCfg->type == META_CHILD_TABLE) { diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 049b69991f..a2454e4c5e 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -13,4 +13,5 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) -add_subdirectory(nodes) \ No newline at end of file +add_subdirectory(nodes) +add_subdirectory(scalar) \ No newline at end of file diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b5c7f44f98..d4da79f968 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1556,39 +1556,43 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons STableMetaOutput *output = NULL; - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + while (true) { + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); - if (CTG_IS_META_TABLE(output->metaType)) { - *pTableMeta = output->tbMeta; - goto _return; - } + if (CTG_IS_META_TABLE(output->metaType)) { + *pTableMeta = output->tbMeta; + goto _return; + } - if (CTG_IS_META_BOTH(output->metaType)) { - memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + if (CTG_IS_META_BOTH(output->metaType)) { + memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + *pTableMeta = output->tbMeta; + goto _return; + } + + if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { + ctgError("invalid metaType:%d", output->metaType); + tfree(output->tbMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + // HANDLE ONLY CHILD TABLE META + + SName stbName = *pTableName; + strcpy(stbName.tname, output->tbName); - *pTableMeta = output->tbMeta; - goto _return; + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); + if (0 == exist) { + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + continue; + } + + memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + break; } - if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { - ctgError("invalid metaType:%d", output->metaType); - tfree(output->tbMeta); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - // HANDLE ONLY CHILD TABLE META - - SName stbName = *pTableName; - strcpy(stbName.tname, output->tbName); - - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); - if (0 == exist) { - ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); - CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST); - } - - memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - _return: tfree(output); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 9b53cc1fbb..12a78134c3 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode + PRIVATE os util common function parser planner qcom vnode scalar nodes ) target_include_directories( diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 232b54554f..23acdf1c33 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -29,9 +29,9 @@ extern "C" { #include "executil.h" #include "executor.h" #include "planner.h" +#include "scalar.h" #include "taosdef.h" #include "tarray.h" -#include "tfilter.h" #include "thash.h" #include "tlockfree.h" #include "tpagedbuf.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f5dc7a82b1..5508b5ecd7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2148,7 +2148,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); + //pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { @@ -2174,7 +2174,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT return TSDB_CODE_SUCCESS; _clean: - destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); + //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); tfree(pRuntimeEnv->pResultRowHashTable); tfree(pRuntimeEnv->keyBuf); tfree(pRuntimeEnv->prevRow); @@ -2212,7 +2212,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); + //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); // destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt index ece84207c7..c24993eb89 100644 --- a/source/libs/executor/test/CMakeLists.txt +++ b/source/libs/executor/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( executorTest - PUBLIC os util common transport gtest taos qcom executor function planner + PRIVATE os util common transport gtest taos qcom executor function planner scalar nodes ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index ebea6755d7..c9657607c2 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -32,6 +32,8 @@ #include "trpc.h" #include "stub.h" #include "executor.h" +#include "tmsg.h" +#include "tname.h" namespace { diff --git a/source/libs/function/inc/tunaryoperator.h b/source/libs/function/inc/tunaryoperator.h index 27784e8487..cd40297e07 100644 --- a/source/libs/function/inc/tunaryoperator.h +++ b/source/libs/function/inc/tunaryoperator.h @@ -20,10 +20,10 @@ extern "C" { #endif -#include "tscalarfunction.h" +//#include "tscalarfunction.h" -typedef void (*_unary_scalar_fn_t)(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput); -_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); +//typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput); +//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index d0d89611c4..1aa5871468 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "tscalarfunction.h" #include "os.h" #include "taosdef.h" #include "tmsg.h" @@ -3221,6 +3220,7 @@ static void diff_function(SqlFunctionCtx *pCtx) { } } +#if 0 char *getArithColumnData(void *param, const char* name, int32_t colId) { SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; @@ -3235,15 +3235,16 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { assert(index >= 0); return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } +#endif static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; + //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; - SScalarFuncParam output = {0}; + SScalarParam output = {0}; output.data = pCtx->pOutput; - evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); + //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c deleted file mode 100644 index 4de11cbe71..0000000000 --- a/source/libs/function/src/tbinoperator.c +++ /dev/null @@ -1,488 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "ttypes.h" -#include "tbinoperator.h" -#include "tcompare.h" - -//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); - -void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - - *pOutput = (double)pLeft[i] + pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - - *pOutput = (double)pLeft[0] + pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - *pOutput = (double)pLeft[i] + pRight[0]; - } - } -} - -typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); - -double getVectorDoubleValue_TINYINT(void *src, int32_t index) { - return (double)*((int8_t *)src + index); -} -double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { - return (double)*((uint8_t *)src + index); -} -double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { - return (double)*((int16_t *)src + index); -} -double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { - return (double)*((uint16_t *)src + index); -} -double getVectorDoubleValue_INT(void *src, int32_t index) { - return (double)*((int32_t *)src + index); -} -double getVectorDoubleValue_UINT(void *src, int32_t index) { - return (double)*((uint32_t *)src + index); -} -double getVectorDoubleValue_BIGINT(void *src, int32_t index) { - return (double)*((int64_t *)src + index); -} -double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { - return (double)*((uint64_t *)src + index); -} -double getVectorDoubleValue_FLOAT(void *src, int32_t index) { - return (double)*((float *)src + index); -} -double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { - return (double)*((double *)src + index); -} -_getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { - _getDoubleValue_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorDoubleValue_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorDoubleValue_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorDoubleValue_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorDoubleValue_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorDoubleValue_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorDoubleValue_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorDoubleValue_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorDoubleValue_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorDoubleValue_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorDoubleValue_DOUBLE; - }else { - assert(0); - } - return p; -} - - -typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index); - -void* getVectorValueAddr_TINYINT(void *src, int32_t index) { - return (void*)((int8_t *)src + index); -} -void* getVectorValueAddr_UTINYINT(void *src, int32_t index) { - return (void*)((uint8_t *)src + index); -} -void* getVectorValueAddr_SMALLINT(void *src, int32_t index) { - return (void*)((int16_t *)src + index); -} -void* getVectorValueAddr_USMALLINT(void *src, int32_t index) { - return (void*)((uint16_t *)src + index); -} -void* getVectorValueAddr_INT(void *src, int32_t index) { - return (void*)((int32_t *)src + index); -} -void* getVectorValueAddr_UINT(void *src, int32_t index) { - return (void*)((uint32_t *)src + index); -} -void* getVectorValueAddr_BIGINT(void *src, int32_t index) { - return (void*)((int64_t *)src + index); -} -void* getVectorValueAddr_UBIGINT(void *src, int32_t index) { - return (void*)((uint64_t *)src + index); -} -void* getVectorValueAddr_FLOAT(void *src, int32_t index) { - return (void*)((float *)src + index); -} -void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { - return (void*)((double *)src + index); -} - -_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { - _getValueAddr_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorValueAddr_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorValueAddr_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorValueAddr_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorValueAddr_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorValueAddr_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorValueAddr_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorValueAddr_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorValueAddr_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorValueAddr_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorValueAddr_DOUBLE; - }else { - assert(0); - } - return p; -} - -void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); - } - } -} - -void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); - } - } -} -void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); - } - } -} - -void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); - } - } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); - } - } -} - -void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - double * output = (double *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); - } - } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - - double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); - } - } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); - } - } -} - -void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t len = pLeft->bytes + pRight->bytes; - - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - char *output = (char *)out; - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += len) { - char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); - char* right = POINTER_SHIFT(pRight->data, pRight->bytes * i); - - if (isNull(left, pLeft->type) || isNull(right, pRight->type)) { - setVardataNull(output, TSDB_DATA_TYPE_BINARY); - continue; - } - - // todo define a macro - memcpy(varDataVal(output), varDataVal(left), varDataLen(left)); - memcpy(varDataVal(output) + varDataLen(left), varDataVal(right), varDataLen(right)); - varDataSetLen(output, varDataLen(left) + varDataLen(right)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += len) { - char *right = POINTER_SHIFT(pRight->data, pRight->bytes * i); - if (isNull(pLeft->data, pLeft->type) || isNull(right, pRight->type)) { - setVardataNull(output, TSDB_DATA_TYPE_BINARY); - continue; - } - - memcpy(varDataVal(output), varDataVal(pLeft->data), varDataLen(pLeft->data)); - memcpy(varDataVal(output) + varDataLen(pLeft->data), varDataVal(right), varDataLen(right)); - varDataSetLen(output, varDataLen(pLeft->data) + varDataLen(right)); - } - } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += len) { - char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); - if (isNull(left, pLeft->type) || isNull(pRight->data, pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - - memcpy(varDataVal(output), varDataVal(left), varDataLen(pRight->data)); - memcpy(varDataVal(output) + varDataLen(left), varDataVal(pRight->data), varDataLen(pRight->data)); - varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); - } - } - -} - -_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { - switch (binFunctionId) { - case TSDB_BINARY_OP_ADD: - return vectorAdd; - case TSDB_BINARY_OP_SUBTRACT: - return vectorSub; - case TSDB_BINARY_OP_MULTIPLY: - return vectorMultiply; - case TSDB_BINARY_OP_DIVIDE: - return vectorDivide; - case TSDB_BINARY_OP_REMAINDER: - return vectorRemainder; - case TSDB_BINARY_OP_CONCAT: - return vectorConcat; - default: - assert(0); - return NULL; - } -} - -bool isBinaryStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 36c9e2513f..e302643c32 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -3,7 +3,6 @@ #include "function.h" #include "thash.h" #include "taggfunction.h" -#include "tscalarfunction.h" static SHashObj* functionHashTable = NULL; static SHashObj* udfHashTable = NULL; @@ -18,12 +17,14 @@ static void doInitFunctionHashTable() { taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } +/* numOfEntries = tListLen(scalarFunc); for(int32_t i = 0; i < numOfEntries; ++i) { int32_t len = (int32_t) strlen(scalarFunc[i].name); SScalarFunctionInfo* ptr = &scalarFunc[i]; taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } +*/ udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true); } diff --git a/source/libs/function/src/tunaryoperator.c b/source/libs/function/src/tunaryoperator.c index 9651f98d08..957f0799c5 100644 --- a/source/libs/function/src/tunaryoperator.c +++ b/source/libs/function/src/tunaryoperator.c @@ -4,10 +4,10 @@ // TODO dynamic define these functions -_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { - assert(0); -} +//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { +// assert(0); +//} -bool isStringOperatorFn(int32_t op) { - return op == FUNCTION_LENGTH; -} +//bool isStringOperatorFn(int32_t op) { +// return op == FUNCTION_LENGTH; +//} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 63bb11a821..78606cd7a2 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -15,6 +15,7 @@ #include "querynodes.h" #include "taos.h" +#include "taoserror.h" #define COPY_SCALAR_FIELD(fldname) \ do { \ @@ -57,14 +58,13 @@ static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) { } static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { - COPY_SCALAR_FIELD(type); dataTypeCopy(&pSrc->resType, &pDst->resType); COPY_CHAR_ARRAY_FIELD(aliasName); // COPY_NODE_LIST_FIELD(pAssociationList); } static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_SCALAR_FIELD(colId); COPY_SCALAR_FIELD(colType); COPY_CHAR_ARRAY_FIELD(dbName); @@ -76,7 +76,7 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); COPY_SCALAR_FIELD(isDuration); switch (pSrc->node.resType.type) { @@ -119,7 +119,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { } static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_SCALAR_FIELD(opType); COPY_NODE_FIELD(pLeft); COPY_NODE_FIELD(pRight); @@ -127,14 +127,14 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { } static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_SCALAR_FIELD(condType); COPY_NODE_LIST_FIELD(pParameterList); return (SNode*)pDst; } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { - exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_ARRAY_FIELD(functionName); COPY_SCALAR_FIELD(funcId); COPY_SCALAR_FIELD(funcType); @@ -142,12 +142,19 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { return (SNode*)pDst; } +static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) { + COPY_SCALAR_FIELD(groupingSetType); + COPY_NODE_LIST_FIELD(pParameterList); + return (SNode*)pDst; +} + SNode* nodesCloneNode(const SNode* pNode) { if (NULL == pNode) { return NULL; } SNode* pDst = nodesMakeNode(nodeType(pNode)); if (NULL == pDst) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } switch (nodeType(pNode)) { @@ -164,7 +171,9 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: + break; case QUERY_NODE_GROUPING_SET: + return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst); case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_LIMIT: default: @@ -174,8 +183,13 @@ SNode* nodesCloneNode(const SNode* pNode) { } SNodeList* nodesCloneList(const SNodeList* pList) { + if (NULL == pList) { + return NULL; + } + SNodeList* pDst = nodesMakeList(); if (NULL == pDst) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SNode* pNode; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d1f32ae314..8968f0579d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -26,27 +26,47 @@ static char* nodeName(ENodeType type) { case QUERY_NODE_COLUMN: return "Column"; case QUERY_NODE_VALUE: + return "Value"; case QUERY_NODE_OPERATOR: + return "Operator"; case QUERY_NODE_LOGIC_CONDITION: + return "LogicCondition"; case QUERY_NODE_FUNCTION: + return "Function"; case QUERY_NODE_REAL_TABLE: + return "RealTable"; case QUERY_NODE_TEMP_TABLE: + return "TempTable"; case QUERY_NODE_JOIN_TABLE: + return "JoinTable"; case QUERY_NODE_GROUPING_SET: + return "GroupingSet"; case QUERY_NODE_ORDER_BY_EXPR: + return "OrderByExpr"; case QUERY_NODE_LIMIT: + return "Limit"; case QUERY_NODE_STATE_WINDOW: + return "StateWindow"; case QUERY_NODE_SESSION_WINDOW: + return "SessionWinow"; case QUERY_NODE_INTERVAL_WINDOW: + return "IntervalWindow"; case QUERY_NODE_NODE_LIST: + return "NodeList"; case QUERY_NODE_FILL: + return "Fill"; case QUERY_NODE_COLUMN_REF: + return "ColumnRef"; case QUERY_NODE_TARGET: + return "Target"; case QUERY_NODE_RAW_EXPR: + return "RawExpr"; case QUERY_NODE_SET_OPERATOR: + return "SetOperator"; case QUERY_NODE_SELECT_STMT: + return "SelectStmt"; case QUERY_NODE_SHOW_STMT: - break; + return "ShowStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -119,7 +139,7 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; -static int32_t logicScanToJson(const void* pObj, SJson* pJson) { +static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -135,7 +155,7 @@ static int32_t logicScanToJson(const void* pObj, SJson* pJson) { static const char* jkProjectLogicPlanProjections = "Projections"; -static int32_t logicProjectToJson(const void* pObj, SJson* pJson) { +static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -149,7 +169,7 @@ static int32_t logicProjectToJson(const void* pObj, SJson* pJson) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; -static int32_t logicJoinToJson(const void* pObj, SJson* pJson) { +static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -163,14 +183,14 @@ static int32_t logicJoinToJson(const void* pObj, SJson* pJson) { return code; } -static int32_t logicFilterToJson(const void* pObj, SJson* pJson) { +static int32_t logicFilterNodeToJson(const void* pObj, SJson* pJson) { return logicPlanNodeToJson(pObj, pJson); } static const char* jkAggLogicPlanGroupKeys = "GroupKeys"; static const char* jkAggLogicPlanAggFuncs = "AggFuncs"; -static int32_t logicAggToJson(const void* pObj, SJson* pJson) { +static int32_t logicAggNodeToJson(const void* pObj, SJson* pJson) { const SAggLogicNode* pNode = (const SAggLogicNode*)pObj; int32_t code = logicPlanNodeToJson(pObj, pJson); @@ -184,17 +204,291 @@ static int32_t logicAggToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkDataTypeType = "Type"; +static const char* jkDataTypePrecision = "Precision"; +static const char* jkDataTypeScale = "Scale"; +static const char* jkDataTypeDataBytes = "Bytes"; + +static int32_t dataTypeToJson(const void* pObj, SJson* pJson) { + const SDataType* pNode = (const SDataType*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkDataTypeType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDataTypePrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDataTypeScale, pNode->scale); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDataTypeDataBytes, pNode->bytes); + } + + return code; +} + +static const char* jkExprDataType = "DataType"; +static const char* jkExprAliasName = "AliasName"; + +static int32_t exprNodeToJson(const void* pObj, SJson* pJson) { + const SExprNode* pNode = (const SExprNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkExprDataType, dataTypeToJson, &pNode->resType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkExprAliasName, pNode->aliasName); + } + + return code; +} + +static const char* jkColumnTableId = "TableId"; +static const char* jkColumnColId = "ColId"; +static const char* jkColumnColType = "ColType"; +static const char* jkColumnDbName = "DbName"; +static const char* jkColumnTableName = "TableName"; +static const char* jkColumnTableAlias = "TableAlias"; +static const char* jkColumnColName = "ColName"; + +static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { + const SColumnNode* pNode = (const SColumnNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnTableId, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnColType, pNode->colType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnTableAlias, pNode->tableAlias); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnColName, pNode->colName); + } + + return code; +} + +// typedef struct SValueNode { +// SExprNode node; // QUERY_NODE_VALUE +// char* ; +// bool ; +// union { +// bool b; +// int64_t i; +// uint64_t u; +// double d; +// char* p; +// } datum; +// } SValueNode; + +static const char* jkValueLiteral = "Literal"; +static const char* jkValueDuration = "Duration"; +static const char* jkValueDatum = "Datum"; + +static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { + const SValueNode* pNode = (const SValueNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->isDuration); + } + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.i); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.u); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + code = tjsonAddDoubleToObject(pJson, jkValueDuration, pNode->datum.d); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->datum.p); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + +static const char* jkOperatorType = "OpType"; +static const char* jkOperatorLeft = "Left"; +static const char* jkOperatorRight = "Right"; + +static int32_t operatorNodeToJson(const void* pObj, SJson* pJson) { + const SOperatorNode* pNode = (const SOperatorNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkOperatorType, pNode->opType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkOperatorLeft, nodeToJson, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkOperatorRight, nodeToJson, pNode->pRight); + } + + return code; +} + +static const char* jkLogicCondType = "CondType"; +static const char* jkLogicCondParameters = "Parameters"; + +static int32_t logicConditionNodeToJson(const void* pObj, SJson* pJson) { + const SLogicConditionNode* pNode = (const SLogicConditionNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicCondType, pNode->condType); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkLogicCondParameters, nodeToJson, pNode->pParameterList); + } + + return code; +} + +static const char* jkFunctionName = "Name"; +static const char* jkFunctionId = "Id"; +static const char* jkFunctionType = "Type"; +static const char* jkFunctionParameter = "Parameters"; + +static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { + const SFunctionNode* pNode = (const SFunctionNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkFunctionName, pNode->functionName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionId, pNode->funcId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionType, pNode->funcType); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkFunctionParameter, nodeToJson, pNode->pParameterList); + } + + return code; +} + +static const char* jkGroupingSetType = "GroupingSetType"; +static const char* jkGroupingSetParameter = "Parameters"; + +static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { + const SGroupingSetNode* pNode = (const SGroupingSetNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkGroupingSetType, pNode->groupingSetType); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkGroupingSetParameter, nodeToJson, pNode->pParameterList); + } + + return code; +} + +static const char* jkSelectStmtDistinct = "Distinct"; +static const char* jkSelectStmtProjections = "Projections"; +static const char* jkSelectStmtFrom = "From"; +static const char* jkSelectStmtWhere = "Where"; +static const char* jkSelectStmtPartitionBy = "PartitionBy"; +static const char* jkSelectStmtWindow = "Window"; +static const char* jkSelectStmtGroupBy = "GroupBy"; +static const char* jkSelectStmtHaving = "Having"; +static const char* jkSelectStmtOrderBy = "OrderBy"; +static const char* jkSelectStmtLimit = "Limit"; +static const char* jkSelectStmtSlimit = "Slimit"; + +static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { + const SSelectStmt* pNode = (const SSelectStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtProjections, nodeToJson, pNode->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtFrom, nodeToJson, pNode->pFromTable); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtWhere, nodeToJson, pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtPartitionBy, nodeToJson, pNode->pPartitionByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtWindow, nodeToJson, pNode->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtGroupBy, nodeToJson, pNode->pGroupByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtHaving, nodeToJson, pNode->pHaving); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkSelectStmtOrderBy, nodeToJson, pNode->pOrderByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtLimit, nodeToJson, pNode->pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSelectStmtSlimit, nodeToJson, pNode->pSlimit); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: + return columnNodeToJson(pObj, pJson); case QUERY_NODE_VALUE: + return valueNodeToJson(pObj, pJson); case QUERY_NODE_OPERATOR: + return operatorNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_CONDITION: + return logicConditionNodeToJson(pObj, pJson); case QUERY_NODE_FUNCTION: + return functionNodeToJson(pObj, pJson); case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: + break; case QUERY_NODE_GROUPING_SET: + return groupingSetNodeToJson(pObj, pJson); case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_LIMIT: case QUERY_NODE_STATE_WINDOW: @@ -206,19 +500,21 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_TARGET: case QUERY_NODE_RAW_EXPR: case QUERY_NODE_SET_OPERATOR: + break; case QUERY_NODE_SELECT_STMT: + return selectStmtTojson(pObj, pJson); case QUERY_NODE_SHOW_STMT: break; case QUERY_NODE_LOGIC_PLAN_SCAN: - return logicScanToJson(pObj, pJson); + return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: - return logicJoinToJson(pObj, pJson); + return logicJoinNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_FILTER: - return logicFilterToJson(pObj, pJson); + return logicFilterNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_AGG: - return logicAggToJson(pObj, pJson); + return logicAggNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PROJECT: - return logicProjectToJson(pObj, pJson); + return logicProjectNodeToJson(pObj, pJson); default: break; } @@ -238,7 +534,7 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) { return code; } -int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { +int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen) { if (NULL == pNode || NULL == pStr || NULL == pLen) { return TSDB_CODE_SUCCESS; } @@ -255,7 +551,7 @@ int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { return code; } - *pStr = tjsonToString(pJson); + *pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson); tjsonDelete(pJson); *pLen = strlen(*pStr) + 1; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 868af93c92..f2205f511e 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -17,6 +17,7 @@ #include "plannodes.h" #include "taos.h" #include "taoserror.h" +#include "taos.h" #include "thash.h" static SNode* makeNode(ENodeType type, size_t size) { @@ -132,11 +133,22 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { } int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { - pTarget->pTail->pNext = pSrc->pHead; - pSrc->pHead->pPrev = pTarget->pTail; + if (NULL == pTarget || NULL == pSrc) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == pTarget->pHead) { + pTarget->pHead = pSrc->pHead; + } else { + pTarget->pTail->pNext = pSrc->pHead; + if (NULL != pSrc->pHead) { + pSrc->pHead->pPrev = pTarget->pTail; + } + } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; tfree(pSrc); + return TSDB_CODE_SUCCESS; } @@ -172,6 +184,36 @@ void nodesDestroyList(SNodeList* pList) { tfree(pList); } +void *nodesGetValueFromNode(SValueNode *pNode) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + return (void *)&pNode->datum.b; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return (void *)&pNode->datum.i; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + return (void *)&pNode->datum.u; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + return (void *)&pNode->datum.d; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + return (void *)pNode->datum.p; + default: + break; + } + + return NULL; +} + bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); @@ -290,6 +332,10 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* nodesDestroyList(cxt.pCols); return cxt.errCode; } + if (0 == LIST_LENGTH(cxt.pCols)) { + nodesDestroyList(cxt.pCols); + cxt.pCols = NULL; + } *pCols = cxt.pCols; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index d353c245cb..cf8d05975b 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -349,7 +349,7 @@ static SNodeList* getProjectList(SNode* pNode) { return NULL; } -static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, SColumnNode* pCol) { +static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, bool isTag, SColumnNode* pCol) { strcpy(pCol->dbName, pTable->table.dbName); strcpy(pCol->tableAlias, pTable->table.tableAlias); strcpy(pCol->tableName, pTable->table.tableName); @@ -359,7 +359,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p } pCol->tableId = pTable->pMeta->uid; pCol->colId = pColSchema->colId; - // pCol->colType = pColSchema->type; + pCol->colType = isTag ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN; pCol->node.resType.type = pColSchema->type; pCol->node.resType.bytes = pColSchema->bytes; } @@ -383,7 +383,7 @@ static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode if (NULL == pCol) { return generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i < pMeta->tableInfo.numOfTags), pCol); nodesListAppend(pList, (SNode*)pCol); } } else { @@ -408,7 +408,7 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i < pMeta->tableInfo.numOfTags), pCol); found = true; break; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index d68b04a384..8ed67d7f2f 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -29,9 +29,9 @@ namespace { void generateTestT1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 4) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20).addColumn("c3", TSDB_DATA_TYPE_BIGINT); builder.done(); } diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index 40a82f4175..54a7330667 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -18,20 +18,20 @@ #define CHECK_ALLOC(p, res) \ do { \ - if (NULL == p) { \ + if (NULL == (p)) { \ printf("%s : %d\n", __FUNCTION__, __LINE__); \ pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ - return res; \ + return (res); \ } \ } while (0) #define CHECK_CODE(exec, res) \ do { \ - int32_t code = exec; \ + int32_t code = (exec); \ if (TSDB_CODE_SUCCESS != code) { \ printf("%s : %d\n", __FUNCTION__, __LINE__); \ pCxt->errCode = code; \ - return res; \ + return (res); \ } \ } while (0) @@ -83,14 +83,30 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - SNode* pNode; - FOREACH(pNode, pExprs) { - if (QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_VALUE == nodeType(pNode)) { - continue; +typedef struct SNameExprCxt { + int32_t planNodeId; + int32_t rewriteId; +} SNameExprCxt; + +static EDealRes doNameExpr(SNode* pNode, void* pContext) { + switch (nodeType(pNode)) { + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { + SNameExprCxt* pCxt = (SNameExprCxt*)pContext; + sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++); + return DEAL_RES_IGNORE_CHILD; } - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", planNodeId, rewriteId); + default: + break; } + + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { + SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId }; + nodesWalkList(pExprs, doNameExpr, &nameCxt); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -135,12 +151,16 @@ static SLogicNode* createScanLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, // set columns to scan SNodeList* pCols = NULL; CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan); - pScan->pScanCols = nodesCloneList(pCols); - CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); + if (NULL != pCols) { + pScan->pScanCols = nodesCloneList(pCols); + CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); + } // set output - pScan->node.pTargets = nodesCloneList(pCols); - CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); + if (NULL != pCols) { + pScan->node.pTargets = nodesCloneList(pCols); + CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); + } return (SLogicNode*)pScan; } @@ -173,8 +193,10 @@ static SLogicNode* createJoinLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); // set on conditions - pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); - CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); + if (NULL != pJoinTable->pOnCond) { + pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); + CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); + } // set the output pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); @@ -220,35 +242,57 @@ static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SLogicNode* pC return (SLogicNode*)pFilter; } -static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) { - SNodeList* pList = nodesMakeList(); - CHECK_ALLOC(pList, NULL); - SNode* pNode; - FOREACH(pNode, pExprs) { - if (QUERY_NODE_VALUE == nodeType(pNode)) { - continue; - } else if (QUERY_NODE_COLUMN == nodeType(pNode)) { +typedef struct SCreateColumnCxt { + int32_t errCode; + SNodeList* pList; +} SCreateColumnCxt; + +static EDealRes doCreateColumn(SNode* pNode, void* pContext) { + SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: { SNode* pCol = nodesCloneNode(pNode); - if (NULL == pCol) { - goto error; + if (NULL == pCol || TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, pCol)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; } - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, pCol)) { - goto error; - } - } else { + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { - goto error; + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); + if (TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, (SNode*)pCol)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + return DEAL_RES_IGNORE_CHILD; } + default: + break; } - return pList; -error: - nodesDestroyList(pList); - return NULL; + + return DEAL_RES_CONTINUE; +} + +static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) { + SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; + if (NULL == cxt.pList) { + return NULL; + } + nodesWalkList(pExprs, doCreateColumn, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pList); + return NULL; + } + return cxt.pList; } static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) { @@ -263,24 +307,37 @@ static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) pAgg->node.id = pCxt->planNodeId++; // set grouyp keys, agg funcs and having conditions - pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); - CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); - pAgg->pAggFuncs = nodesCloneList(pAggFuncs); - CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); + if (NULL != pSelect->pGroupByList) { + pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); + CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); + } + if (NULL != pAggFuncs) { + pAgg->pAggFuncs = nodesCloneList(pAggFuncs); + CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); + } // rewrite the expression in subsequent clauses CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); - pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); - CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); + if (NULL != pSelect->pHaving) { + pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); + CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); + } // set the output - pAgg->node.pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); + pAgg->node.pTargets = nodesMakeList(); CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); - SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); - CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); - nodesListAppendList(pAgg->node.pTargets, pTargets); + if (NULL != pAgg->pGroupKeys) { + SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); + CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); + nodesListAppendList(pAgg->node.pTargets, pTargets); + } + if (NULL != pAgg->pAggFuncs) { + SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); + CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); + nodesListAppendList(pAgg->node.pTargets, pTargets); + } return (SLogicNode*)pAgg; } @@ -344,7 +401,7 @@ static SLogicNode* createQueryLogicNode(SPlanContext* pCxt, SNode* pStmt) { } int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) { - SPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 0 }; + SPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1 }; SLogicNode* pRoot = createQueryLogicNode(&cxt, pNode); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode((SNode*)pRoot); diff --git a/source/libs/planner/test/newPlannerTest.cpp b/source/libs/planner/test/newPlannerTest.cpp index c227bf88ba..c3ee4fb9a8 100644 --- a/source/libs/planner/test/newPlannerTest.cpp +++ b/source/libs/planner/test/newPlannerTest.cpp @@ -42,26 +42,28 @@ protected: bool run() { int32_t code = parser(&cxt_, &query_); - // cout << "parser return " << code << endl; + if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] parser code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + cout << "sql:[" << cxt_.pSql << "] parser code:" << code << ", strerror:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; return false; } + + const string syntaxTreeStr = toString(query_.pRoot, false); + SLogicNode* pLogicPlan = nullptr; code = createLogicPlan(query_.pRoot, &pLogicPlan); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] plan code:" << tstrerror(code) << endl; + cout << "sql:[" << cxt_.pSql << "] plan code:" << code << ", strerror:" << tstrerror(code) << endl; return false; } - char* pStr = NULL; - int32_t len = 0; - code = nodesNodeToString((const SNode*)pLogicPlan, &pStr, &len); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] toString code:" << tstrerror(code) << endl; - return false; - } - cout << "logic plan : " << endl; - cout << pStr << endl; + + cout << "sql : [" << cxt_.pSql << "]" << endl; + cout << "syntax test : " << endl; + cout << syntaxTreeStr << endl; + // cout << "logic plan : " << endl; + // cout << toString((const SNode*)pLogicPlan) << endl; + cout << "unformatted logic plan : " << endl; + cout << toString((const SNode*)pLogicPlan, false) << endl; return true; } @@ -75,6 +77,19 @@ private: cxt_.msgLen = max_err_len; } + string toString(const SNode* pRoot, bool format = true) { + char* pStr = NULL; + int32_t len = 0; + int32_t code = nodesNodeToString(pRoot, format, &pStr, &len); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; + return string(); + } + string str(pStr); + tfree(pStr); + return str; + } + string acctId_; string db_; char errMagBuf_[max_err_len]; @@ -89,3 +104,19 @@ TEST_F(NewPlannerTest, simple) { bind("SELECT * FROM t1"); ASSERT_TRUE(run()); } + +TEST_F(NewPlannerTest, groupBy) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); + ASSERT_TRUE(run()); + + bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); + ASSERT_TRUE(run()); + + bind("SELECT c1 + c3, count(*) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt new file mode 100644 index 0000000000..d2d02bc0dc --- /dev/null +++ b/source/libs/scalar/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src SCALAR_SRC) + +add_library(scalar STATIC ${SCALAR_SRC}) +target_include_directories( + scalar + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(scalar + PRIVATE os util common nodes function qcom + ) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/inc/tfilter.h b/source/libs/scalar/inc/filterInt.h similarity index 73% rename from source/libs/executor/inc/tfilter.h rename to source/libs/scalar/inc/filterInt.h index 55edf27949..1dd533c1c5 100644 --- a/source/libs/executor/inc/tfilter.h +++ b/source/libs/scalar/inc/filterInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_QFILTER_H -#define TDENGINE_QFILTER_H +#ifndef TDENGINE_FILTER_INT_H +#define TDENGINE_FILTER_INT_H #ifdef __cplusplus extern "C" { @@ -22,7 +22,10 @@ extern "C" { #include "thash.h" #include "tname.h" -#include "function.h" +#include "common.h" +#include "scalar.h" +#include "querynodes.h" +#include "query.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -83,6 +86,12 @@ enum { RANGE_TYPE_MR_CTX = 3, }; +enum { + FI_ACTION_NO_NEED = 1, + FI_ACTION_CONTINUE, + FI_ACTION_STOP, +}; + typedef struct OptrStr { uint16_t optr; char *str; @@ -95,17 +104,11 @@ typedef struct SFilterRange { char eflag; } SFilterRange; -typedef struct SFilterColRange { - uint16_t idx; //column field idx - bool isNull; - bool notNull; - bool isRange; - SFilterRange ra; -} SFilterColRange; - typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); +typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); +typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { int64_t s; @@ -153,37 +156,39 @@ typedef struct SFilterField { } SFilterField; typedef struct SFilterFields { - uint16_t size; - uint16_t num; + uint32_t size; + uint32_t num; SFilterField *fields; } SFilterFields; typedef struct SFilterFieldId { uint16_t type; - uint16_t idx; + uint32_t idx; } SFilterFieldId; typedef struct SFilterGroup { - uint16_t unitSize; - uint16_t unitNum; - uint16_t *unitIdxs; + uint32_t unitSize; + uint32_t unitNum; + uint32_t *unitIdxs; uint8_t *unitFlags; // !unit result } SFilterGroup; typedef struct SFilterColInfo { uint8_t type; int32_t dataType; + uint8_t optr; // for equal operation in the relation of RELATION_IN + int64_t value; // for equal operation in the relation of RELATION_IN void *info; } SFilterColInfo; typedef struct SFilterGroupCtx { - uint16_t colNum; - uint16_t *colIdx; + uint32_t colNum; + uint32_t *colIdx; SFilterColInfo *colInfo; } SFilterGroupCtx; typedef struct SFilterColCtx { - uint16_t colIdx; + uint32_t colIdx; void* ctx; } SFilterColCtx; @@ -217,16 +222,32 @@ typedef struct SFilterPCtx { SHashObj *unitHash; } SFilterPCtx; +typedef struct SFltTreeStat { + int32_t code; + bool scalarMode; +} SFltTreeStat; + +typedef struct SFltScalarCtx { + SNode *node; +} SFltScalarCtx; + +typedef struct SFltBuildGroupCtx { + SFilterInfo *info; + SArray *group; + int32_t code; +} SFltBuildGroupCtx; + typedef struct SFilterInfo { + bool scalarMode; + SFltScalarCtx sclCtx; uint32_t options; uint32_t status; - uint16_t unitSize; - uint16_t unitNum; - uint16_t groupNum; - uint16_t colRangeNum; + uint32_t unitSize; + uint32_t unitNum; + uint32_t groupNum; + uint32_t colRangeNum; SFilterFields fields[FLD_TYPE_MAX]; SFilterGroup *groups; - uint16_t *cgroups; SFilterUnit *units; SFilterComUnit *cunits; uint8_t *unitRes; // result @@ -234,16 +255,15 @@ typedef struct SFilterInfo { SFilterRangeCtx **colRange; filter_exec_func func; uint8_t blkFlag; - uint16_t blkGroupNum; - uint16_t *blkUnits; + uint32_t blkGroupNum; + uint32_t *blkUnits; int8_t *blkUnitRes; - + void *pTable; + SFilterPCtx pctx; } SFilterInfo; -#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) - -#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) #define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) @@ -259,7 +279,7 @@ typedef struct SFilterInfo { #define FILTER_CLR_FLAG(st, f) st &= (~f) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) -#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) +#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0) #define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) #define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) @@ -269,25 +289,33 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) -#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) -#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) -#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) +#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL) + +#define fltFatal(...) qFatal(__VA_ARGS__) +#define fltError(...) qError(__VA_ARGS__) +#define fltWarn(...) qWarn(__VA_ARGS__) +#define fltInfo(...) qInfo(__VA_ARGS__) +#define fltDebug(...) qDebug(__VA_ARGS__) +#define fltTrace(...) qTrace(__VA_ARGS__) + + +#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0) +#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define CHK_RETV(c) do { if (c) { return; } } while (0) -#define CHK_RET(c, r) do { if (c) { return r; } } while (0) -#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) -#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) -#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) -#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) +#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) +#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -300,6 +328,7 @@ typedef struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) +#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) @@ -314,7 +343,7 @@ typedef struct SFilterInfo { #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) @@ -322,18 +351,25 @@ typedef struct SFilterInfo { #define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) - -extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); +#if 0 +extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); +extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp); +extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); +#else +//REMOVE THESE!!!!!!!!!!!!!!!!!!!! +#include "function.h" +#endif +extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); +extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); #ifdef __cplusplus } #endif -#endif // TDENGINE_QFILTER_H +#endif // TDENGINE_FILTER_INT_H diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h new file mode 100644 index 0000000000..3cfea6890f --- /dev/null +++ b/source/libs/scalar/inc/sclInt.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_SCALARINT_H +#define TDENGINE_SCALARINT_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "common.h" +#include "thash.h" +#include "query.h" + +typedef struct SScalarCtx { + int32_t code; + SSDataBlock *pSrc; + SHashObj *pRes; /* element is SScalarParam */ +} SScalarCtx; + +#define SCL_DEFAULT_OP_NUM 10 + +#define sclFatal(...) qFatal(__VA_ARGS__) +#define sclError(...) qError(__VA_ARGS__) +#define sclWarn(...) qWarn(__VA_ARGS__) +#define sclInfo(...) qInfo(__VA_ARGS__) +#define sclDebug(...) qDebug(__VA_ARGS__) +#define sclTrace(...) qTrace(__VA_ARGS__) + +#define SCL_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SCALARINT_H \ No newline at end of file diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/scalar/inc/sclfunc.h similarity index 89% rename from source/libs/function/inc/tscalarfunction.h rename to source/libs/scalar/inc/sclfunc.h index 6d23775610..8e03470033 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/scalar/inc/sclfunc.h @@ -20,13 +20,7 @@ extern "C" { #endif #include "function.h" - -typedef struct SScalarFuncParam { - void* data; - int32_t num; - int32_t type; - int32_t bytes; -} SScalarFuncParam; +#include "scalar.h" typedef struct SScalarFunctionSupport { struct SExprInfo *pExprInfo; @@ -39,7 +33,7 @@ typedef struct SScalarFunctionSupport { extern struct SScalarFunctionInfo scalarFunc[8]; -int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, +int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); diff --git a/source/libs/function/inc/tbinoperator.h b/source/libs/scalar/inc/sclvector.h similarity index 86% rename from source/libs/function/inc/tbinoperator.h rename to source/libs/scalar/inc/sclvector.h index c1b6b0bc31..69800a54ea 100644 --- a/source/libs/function/inc/tbinoperator.h +++ b/source/libs/scalar/inc/sclvector.h @@ -20,9 +20,9 @@ extern "C" { #endif -#include "tscalarfunction.h" +#include "sclfunc.h" -typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order); +typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); bool isBinaryStringOp(int32_t op); diff --git a/source/libs/executor/src/tfilter.c b/source/libs/scalar/src/filter.c similarity index 72% rename from source/libs/executor/src/tfilter.c rename to source/libs/scalar/src/filter.c index 97dccb5c7b..8f8fc25d18 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/scalar/src/filter.c @@ -17,7 +17,8 @@ #include "thash.h" //#include "queryLog.h" #include "tcompare.h" -#include "tfilter.h" +#include "filterInt.h" +#include "filter.h" OptrStr gOptrStr[] = { {TSDB_RELATION_INVALID, "invalid"}, @@ -28,48 +29,35 @@ OptrStr gOptrStr[] = { {TSDB_RELATION_GREATER_EQUAL, ">="}, {TSDB_RELATION_NOT_EQUAL, "!="}, {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_MATCH, "nmatch"}, {TSDB_RELATION_ISNULL, "is null"}, {TSDB_RELATION_NOTNULL, "not null"}, {TSDB_RELATION_IN, "in"}, {TSDB_RELATION_AND, "and"}, {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"} -}; - -static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { - const SSchema *sch1 = desc1; - const SSchema *sch2 = desc2; - - return sch1->colId != sch2->colId; -} - -static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { - const SVariant *val1 = desc1; - const SVariant *val2 = desc2; - - return taosVariantCompare(val1, val2); -} - - -filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { - NULL, - filterFieldColDescCompare, - filterFieldValDescCompare + {TSDB_RELATION_NOT, "not"}, + {TSDB_RELATION_MATCH, "match"}, + {TSDB_RELATION_NMATCH, "nmatch"} }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0; + int32_t result = cfunc(maxv, minr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0; + int32_t result = cfunc(maxv, minr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) <= 0; + int32_t result = cfunc(minv, maxr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) < 0; + int32_t result = cfunc(minv, maxr); + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; @@ -156,9 +144,11 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { } __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, - compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, + compareDoubleVal, compareLenPrefixedStr, compareStrPatternMatch, compareChkInString, compareWStrPatternMatch, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, - setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch + setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, + compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { @@ -186,6 +176,29 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { assert(0); } } + + if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return 21; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return 22; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return 23; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return 24; + default: + assert(0); + } + } switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -203,8 +216,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 20; } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ comparFn = 7; + } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = 26; } else if (optr == TSDB_RELATION_IN) { comparFn = 8; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = 25; } else { /* normal relational comparFn */ comparFn = 6; } @@ -219,8 +236,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 20; } else if (optr == TSDB_RELATION_LIKE) { comparFn = 9; + } else if (optr == TSDB_RELATION_LIKE) { + comparFn = 27; } else if (optr == TSDB_RELATION_IN) { comparFn = 8; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = 25; } else { comparFn = 10; } @@ -240,6 +261,10 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { return comparFn; } +__compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) { + return gDataCompare[filterGetCompFuncIdx(type, optr)]; +} + static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); @@ -254,7 +279,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) { info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); + info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); @@ -281,7 +306,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt void* filterInitRangeCtx(int32_t type, int32_t options) { if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - //qError("not supported range type:%d", type); + qError("not supported range type:%d", type); return NULL; } @@ -648,7 +673,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) { } if (num == 0) { - //qError("no range result"); + qError("no range result"); return TSDB_CODE_QRY_APP_ERROR; } @@ -738,8 +763,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { int32_t leftSize = (int32_t)taosArrayGetSize(left); int32_t rightSize = (int32_t)taosArrayGetSize(right); -// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); -// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + if (taosArrayGetSize(left) <= 0) { + fltDebug("empty group"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (taosArrayGetSize(right) <= 0) { + fltDebug("empty group"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } for (int32_t l = 0; l < leftSize; ++l) { SFilterGroup *gp1 = taosArrayGet(left, l); @@ -756,8 +788,8 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { } int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { - for (uint16_t i = 0; i < fields->num; ++i) { - if (0 == gDescCompare[type](fields->fields[i].desc, v)) { + for (uint32_t i = 0; i < fields->num; ++i) { + if (nodesEqualNode(fields->fields[i].desc, v)) { return i; } } @@ -769,7 +801,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { if (type == FLD_TYPE_VALUE) { if (info->pctx.valHash == false) { - //qError("value hash is empty"); + qError("value hash is empty"); return -1; } @@ -782,10 +814,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d return -1; } - +// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0 +// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t idx = -1; - uint16_t *num; + uint32_t *num; num = &info->fields[type].num; @@ -847,21 +880,25 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte } -int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { -// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); -// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR); +int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { + if (node == NULL) { + fltError("empty node"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) { + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } int32_t type; void *v; - if (node->nodeType == TEXPR_BINARYEXPR_NODE) { + if (nodeType(node) == QUERY_NODE_COLUMN_REF) { type = FLD_TYPE_COLUMN; - v = node->pSchema; - node->pSchema = NULL; + v = node; } else { type = FLD_TYPE_VALUE; - v = node->pVal; - node->pVal = NULL; + v = node; } filterAddField(info, v, NULL, type, fid, 0, true); @@ -869,7 +906,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI return TSDB_CODE_SUCCESS; } -int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { if (info->pctx.unitHash == NULL) { info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); @@ -878,14 +915,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); if (hu) { - *uidx = *(uint16_t *)hu; + *uidx = *(uint32_t *)hu; return TSDB_CODE_SUCCESS; } } } if (info->unitNum >= info->unitSize) { - uint16_t psize = info->unitSize; + uint32_t psize = info->unitSize; info->unitSize += FILTER_DEFAULT_UNIT_SIZE; info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); @@ -903,7 +940,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { - assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); + if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){ + return -1; + } } SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); @@ -926,7 +965,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi -int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { +int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { group->unitSize += FILTER_DEFAULT_UNIT_SIZE; group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); @@ -937,258 +976,38 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { return TSDB_CODE_SUCCESS; } -int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { - SBufferReader br = tbufInitReader(buf, len, false); - uint32_t sType = tbufReadUint32(&br); - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); +int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { + SOperatorNode *node = (SOperatorNode *)tree; + int32_t ret = TSDB_CODE_SUCCESS; + SFilterFieldId left = {0}, right = {0}; + filterAddFieldFromNode(info, node->pLeft, &left); + uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + int32_t len = 0; + uint32_t uidx = 0; int32_t code = 0; - int dummy = -1; - SVariant tmpVar = {0}; - size_t t = 0; - int32_t sz = tbufReadInt32(&br); - void *pvar = NULL; - int64_t val = 0; - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(sType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = 128; - } - - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { + SNodeListNode *listNode = (SNodeListNode *)node->pRight; + void *fdata = NULL; + SListCell *cell = listNode->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; - for (int32_t i = 0; i < sz; i++) { - switch (sType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)&val = tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(float *)&val = (float)tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_BINARY: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - default: - taosHashCleanup(pObj); - *q = NULL; - assert(0); - } - - taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); + for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + out.data = malloc(sizeof(int64_t)); - if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); - bufLen = (int32_t)t; - } - - bool converted = false; - char extInfo = 0; - - switch (tType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_FLOAT: { -// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } - -// goto _return; -// } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_BINARY: { - if (taosVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - if (taosVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - default: - goto _return; - } - - taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); - taosVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - } - - *q = (void *)pObj; - pObj = NULL; - -_return: - taosVariantDestroy(&tmpVar); - taosHashCleanup(pObj); - tfree(tmp); - - return code; -} - - - -int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { - SFilterFieldId left = {0}, right = {0}; - - filterAddFieldFromNode(info, tree->_node.pLeft, &left); - - SVariant* var = tree->_node.pRight->pVal; - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); - size_t len = 0; - uint16_t uidx = 0; - - if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { - void *data = NULL; - filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); -// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); - - if (taosHashGetSize((SHashObj *)data) <= 0) { - filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - taosHashCleanup(data); - - return TSDB_CODE_SUCCESS; - } - - void *p = taosHashIterate((SHashObj *)data, NULL); - while(p) { - void* key = NULL; - len = 0; - - taosHashGetKey(p, &key, &len); - void *fdata = NULL; - - if (IS_VAR_DATA_TYPE(type)) { - fdata = malloc(len + VARSTR_HEADER_SIZE); - varDataLen(fdata) = len; - memcpy(varDataVal(fdata), key, len); - len += VARSTR_HEADER_SIZE; - } else { - fdata = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(fdata, key); - len = tDataTypes[type].bytes; + code = vectorConvertImpl(&in, &out); + if (code) { + fltError("convert from %d to %d failed", in.type, out.type); + tfree(out.data); + FLT_ERR_RET(code); } + len = tDataTypes[type].bytes; + filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); @@ -1197,15 +1016,11 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g filterAddUnitToGroup(&fgroup, uidx); taosArrayPush(group, &fgroup); - - p = taosHashIterate((SHashObj *)data, p); } - - taosHashCleanup(data); } else { - filterAddFieldFromNode(info, tree->_node.pRight, &right); + filterAddFieldFromNode(info, node->pRight, &right); - filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + FLT_ERR_RET(filterAddUnit(info, node->opType, &left, &right, &uidx)); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); @@ -1217,7 +1032,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g } -int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { +int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; int32_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = FLD_DESC_NO_FREE; @@ -1230,7 +1045,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u void *data = FILTER_UNIT_VAL_DATA(src, u); if (IS_VAR_DATA_TYPE(type)) { if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); @@ -1252,7 +1067,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); } -int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { +int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) { SFilterUnit *u = &info->units[uidx]; u->compare.optr2 = optr; @@ -1262,9 +1077,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ } -int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { +int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { SFilterFieldId left, right, right2; - uint16_t uidx = 0; + uint32_t uidx = 0; SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); @@ -1442,61 +1257,80 @@ static void filterFreeGroup(void *pItem) { } -int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { +EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { int32_t code = TSDB_CODE_SUCCESS; - SArray* leftGroup = NULL; - SArray* rightGroup = NULL; - - if (tree->nodeType != TEXPR_BINARYEXPR_NODE) { - //qError("invalid nodeType:%d", tree->nodeType); - return TSDB_CODE_QRY_APP_ERROR; + SArray* preGroup = NULL; + SArray* newGroup = NULL; + SArray* resGroup = NULL; + ENodeType nType = nodeType(nType); + SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext; + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + SLogicConditionNode *node = (SLogicConditionNode *)pNode; + if (LOGIC_COND_TYPE_AND == node->condType) { + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + newGroup = taosArrayInit(4, sizeof(SFilterGroup)); + resGroup = taosArrayInit(4, sizeof(SFilterGroup)); + + SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; + nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + FLT_ERR_JRET(tctx.code); + + FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); + + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + + preGroup = resGroup; + } + + taosArrayAddAll(ctx->group, resGroup); + + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + taosArrayDestroyEx(resGroup, filterFreeGroup); + + return DEAL_RES_IGNORE_CHILD; + } + + if (LOGIC_COND_TYPE_OR == node->condType) { + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + FLT_ERR_JRET(ctx->code); + } + + return DEAL_RES_IGNORE_CHILD; + } + + fltError("invalid condition type, type:%d", node->condType); + + return DEAL_RES_ERROR; } - - if (tree->_node.optr == TSDB_RELATION_AND) { - leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); - rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); - ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); + if (QUERY_NODE_OPERATOR == nType) { + FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group)); - return TSDB_CODE_SUCCESS; + return DEAL_RES_CONTINUE; } - if (tree->_node.optr == TSDB_RELATION_OR) { - ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); - ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); - - return TSDB_CODE_SUCCESS; - } - - code = filterAddGroupUnitFromNode(info, tree, group); + fltError("invalid node type for filter, type:%d", nodeType(pNode)); + code = TSDB_CODE_QRY_INVALID_INPUT; _return: - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return code; + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + taosArrayDestroyEx(resGroup, filterFreeGroup); + + ctx->code = code; + + return DEAL_RES_ERROR; } -#if 0 -int32_t filterInitUnitFunc(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - - info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - } - - return TSDB_CODE_SUCCESS; -} -#endif - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { +int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { int32_t n = 0; switch (type) { @@ -1574,34 +1408,38 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { if (qDebugFlag & DEBUG_DEBUG) { -// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); + if (info == NULL) { + fltDebug("%s - FilterInfo: EMPTY", msg); + return; + } if (options == 0) { -// //qDebug("%s - FilterInfo:", msg); -// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + qDebug("%s - FilterInfo:", msg); + qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema *sch = field->desc; -// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + SColumnRefNode *refNode = (SColumnRefNode *)field->desc; + qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); } - //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { + qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); + for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { - SVariant *var = field->desc; - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); + SValueNode *var = (SValueNode *)field->desc; + SDataType *dType = &var->node.resType; + if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { + qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); } else { - //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, dType->type, var->datum.i); //TODO } } else if (field->data) { - //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO } } - //qDebug("UNIT Num:%u", info->unitNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { + qDebug("UNIT Num:%u", info->unitNum); + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; int32_t type = FILTER_UNIT_DATA_TYPE(unit); int32_t len = 0; @@ -1609,8 +1447,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SSchema *sch = left->desc; - len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + SColumnRefNode *refNode = (SColumnRefNode *)left->desc; + if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_NMATCH){ + len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); + } if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); @@ -1619,7 +1459,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); + if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); } else { strcat(str, "NULL"); } @@ -1627,7 +1467,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_NMATCH){ + sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); + } if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); @@ -1636,23 +1478,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); } else { strcat(str, "NULL"); } strcat(str, "]"); } - //qDebug("%s", str); //TODO + qDebug("%s", str); //TODO } - //qDebug("GROUP Num:%u", info->groupNum); - for (uint16_t i = 0; i < info->groupNum; ++i) { + qDebug("GROUP Num:%u", info->groupNum); + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup *group = &info->groups[i]; - //qDebug("Group%d : unit num[%u]", i, group->unitNum); + qDebug("Group%d : unit num[%u]", i, group->unitNum); - for (uint16_t u = 0; u < group->unitNum; ++u) { - //qDebug("unit id:%u", group->unitIdxs[u]); + for (uint32_t u = 0; u < group->unitNum; ++u) { + qDebug("unit id:%u", group->unitIdxs[u]); } } @@ -1660,12 +1502,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) } if (options == 1) { - //qDebug("%s - RANGE info:", msg); + qDebug("%s - RANGE info:", msg); - //qDebug("RANGE Num:%u", info->colRangeNum); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + qDebug("RANGE Num:%u", info->colRangeNum); + for (uint32_t i = 0; i < info->colRangeNum; ++i) { SFilterRangeCtx *ctx = info->colRange[i]; - //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); if (ctx->isrange) { SFilterRangeNode *r = ctx->rs; while (r) { @@ -1675,7 +1517,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str,"(NULL)"); } else { FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); } strcat(str, " - "); @@ -1683,10 +1525,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str, "(NULL)"); } else { FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); } - //qDebug("range: %s", str); + qDebug("range: %s", str); r = r->next; } @@ -1696,25 +1538,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) return; } - //qDebug("%s - Block Filter info:", msg); + qDebug("%s - Block Filter info:", msg); if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - //qDebug("Flag:%s", "ALL"); + qDebug("Flag:%s", "ALL"); return; } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - //qDebug("Flag:%s", "EMPTY"); + qDebug("Flag:%s", "EMPTY"); return; } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ - //qDebug("Flag:%s", "ACTIVE"); + qDebug("Flag:%s", "ACTIVE"); } - //qDebug("GroupNum:%d", info->blkGroupNum); - uint16_t *unitIdx = info->blkUnits; - for (uint16_t i = 0; i < info->blkGroupNum; ++i) { - //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); - uint16_t unitNum = *(unitIdx++); - for (uint16_t m = 0; m < unitNum; ++m) { - //qDebug("uidx[%d]", *(unitIdx++)); + qDebug("GroupNum:%d", info->blkGroupNum); + uint32_t *unitIdx = info->blkUnits; + for (uint32_t i = 0; i < info->blkGroupNum; ++i) { + qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); + uint32_t unitNum = *(unitIdx++); + for (uint32_t m = 0; m < unitNum; ++m) { + qDebug("uidx[%d]", *(unitIdx++)); } } } @@ -1800,21 +1642,23 @@ void filterFreePCtx(SFilterPCtx *pctx) { } void filterFreeInfo(SFilterInfo *info) { - CHK_RETV(info == NULL); + if (info == NULL) { + return; + } tfree(info->cunits); tfree(info->blkUnitRes); tfree(info->blkUnits); for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { - for (uint16_t f = 0; f < info->fields[i].num; ++f) { + for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); } tfree(info->fields[i].fields); } - for (int32_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { filterFreeGroup(&info->groups[i]); } @@ -1826,7 +1670,7 @@ void filterFreeInfo(SFilterInfo *info) { tfree(info->unitFlags); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + for (uint32_t i = 0; i < info->colRangeNum; ++i) { filterFreeRangeCtx(info->colRange[i]); } @@ -1862,9 +1706,72 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { return TSDB_CODE_SUCCESS; } +int32_t fltGenerateSetFromList(void **data, void *pNode, uint32_t type) { + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + if (NULL == pObj) { + fltError("taosHashInit failed, size:%d", 256); + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } -int32_t filterInitValFieldData(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + + int32_t code = 0; + SNodeListNode *nodeList = (SNodeListNode *)pNode; + SListCell *cell = nodeList->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; + int8_t dummy = 0; + int32_t bufLen = 60; + out.data = malloc(bufLen); + int32_t len = 0; + void *buf = NULL; + + for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + + if (valueNode->node.resType.type != type) { + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + + code = vectorConvertImpl(&in, &out); + if (code) { + fltError("convert from %d to %d failed", in.type, out.type); + FLT_ERR_JRET(code); + } + + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(out.data); + } else { + len = tDataTypes[type].bytes; + } + + buf = out.data; + } else { + buf = nodesGetValueFromNode(valueNode); + len = valueNode->node.resType.bytes; + } + + if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { + fltError("taosHashPut failed"); + FLT_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + tfree(out.data); + *data = pObj; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(out.data); + taosHashCleanup(pObj); + + FLT_RET(code); +} + +int32_t fltInitValFieldData(SFilterInfo *info) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); @@ -1878,7 +1785,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) { uint32_t type = FILTER_UNIT_DATA_TYPE(unit); SFilterField* fi = right; - SVariant* var = fi->desc; + SValueNode* var = (SValueNode *)fi->desc; if (var == NULL) { assert(fi->data != NULL); @@ -1886,45 +1793,65 @@ int32_t filterInitValFieldData(SFilterInfo *info) { } if (unit->compare.optr == TSDB_RELATION_IN) { - filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); -// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + FLT_ERR_RET(fltGenerateSetFromList((void **)&fi->data, fi->desc, type)); + if (fi->data == NULL) { + fltError("failed to convert in param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); continue; } + SDataType *dType = &var->node.resType; + if (type == TSDB_DATA_TYPE_BINARY) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); } else if (type == TSDB_DATA_TYPE_NCHAR) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - } else { - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE - fi->data = calloc(var->nLen, tDataTypes[type].bytes); - for (int32_t a = 0; a < var->nLen; ++a) { + } else if (type != TSDB_DATA_TYPE_JSON){ + if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE +/* + fi->data = calloc(dType->bytes, tDataTypes[type].bytes); + for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); } - +*/ continue; } else { fi->data = calloc(1, sizeof(int64_t)); } + } else{ // type == TSDB_DATA_TYPE_JSON + // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) } - bool converted = false; - char extInfo = 0; -// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { -// if (converted) { -// filterHandleValueExtInfo(unit, extInfo); -// -// continue; -// } -// //qError("dump value to type[%d] failed", type); -// return TSDB_CODE_TSC_INVALID_OPERATION; -// } + if(type != TSDB_DATA_TYPE_JSON){ + bool converted = false; + char extInfo = 0; + SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; + SScalarParam out = {.data = fi->data, .num = 1, .type = type}; + if (vectorConvertImpl(&in, &out)) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + // match/nmatch for nchar type need convert from ucs4 to mbs + if(type == TSDB_DATA_TYPE_NCHAR && + (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + if (len < 0){ + qError("filterInitValFieldData taosUcs4ToMbs error 1"); + return TSDB_CODE_FAILED; + } + varDataSetLen(newValData, len); + varDataCopy(fi->data, newValData); + } } return TSDB_CODE_SUCCESS; @@ -2025,25 +1952,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c } int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { - CHK_JMP(ctx1->status != ctx2->status); - CHK_JMP(ctx1->isnull != ctx2->isnull); - CHK_JMP(ctx1->notnull != ctx2->notnull); - CHK_JMP(ctx1->isrange != ctx2->isrange); + FLT_CHK_JMP(ctx1->status != ctx2->status); + FLT_CHK_JMP(ctx1->isnull != ctx2->isnull); + FLT_CHK_JMP(ctx1->notnull != ctx2->notnull); + FLT_CHK_JMP(ctx1->isrange != ctx2->isrange); SFilterRangeNode *r1 = ctx1->rs; SFilterRangeNode *r2 = ctx2->rs; while (r1 && r2) { - CHK_JMP(r1->ra.sflag != r2->ra.sflag); - CHK_JMP(r1->ra.eflag != r2->ra.eflag); - CHK_JMP(r1->ra.s != r2->ra.s); - CHK_JMP(r1->ra.e != r2->ra.e); + FLT_CHK_JMP(r1->ra.sflag != r2->ra.sflag); + FLT_CHK_JMP(r1->ra.eflag != r2->ra.eflag); + FLT_CHK_JMP(r1->ra.s != r2->ra.s); + FLT_CHK_JMP(r1->ra.e != r2->ra.e); r1 = r1->next; r2 = r2->next; } - CHK_JMP(r1 != r2); + FLT_CHK_JMP(r1 != r2); *equal = true; @@ -2055,7 +1982,7 @@ _return: } -int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { +int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) { SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; int32_t size = (int32_t)taosArrayGetSize(colArray); int32_t type = gRes->colInfo[colIdx].dataType; @@ -2066,11 +1993,15 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI uint8_t optr = FILTER_UNIT_OPTR(u); filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); - CHK_JMP(*empty); + FLT_CHK_JMP(*empty); if (!FILTER_NO_MERGE_OPTR(optr)) { filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); - CHK_JMP(MR_EMPTY_RES(ctx)); + FLT_CHK_JMP(MR_EMPTY_RES(ctx)); + } + if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ + gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL; + SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u)); } } @@ -2092,11 +2023,11 @@ _return: int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { bool empty = false; - uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); - uint16_t colIdxi = 0; - uint16_t gResIdx = 0; + uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); + uint32_t colIdxi = 0; + uint32_t gResIdx = 0; - for (uint16_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup* g = info->groups + i; gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); @@ -2104,9 +2035,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t colIdxi = 0; empty = false; - for (uint16_t j = 0; j < g->unitNum; ++j) { + for (uint32_t j = 0; j < g->unitNum; ++j) { SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); - uint16_t cidx = FILTER_UNIT_COL_IDX(u); + uint32_t cidx = FILTER_UNIT_COL_IDX(u); if (gRes[gResIdx]->colInfo[cidx].info == NULL) { gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); @@ -2122,10 +2053,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } if (colIdxi > 1) { - qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); } - for (uint16_t l = 0; l < colIdxi; ++l) { + for (uint32_t l = 0; l < colIdxi; ++l) { int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; if (FILTER_NO_MERGE_DATA_TYPE(type)) { @@ -2164,7 +2095,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool equal = false; for (; m < gRes1->colNum; ++m) { @@ -2187,6 +2118,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict = true; return; } + + // for long in operation + if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) { + SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info; + if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){ + *conflict = true; + return; + } + } ++n; equal = true; @@ -2204,7 +2144,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool } -int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { +int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); @@ -2234,10 +2174,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; bool equal = false; - uint16_t equal1 = 0, equal2 = 0, merNum = 0; + uint32_t equal1 = 0, equal2 = 0, merNum = 0; SFilterRangeCtx *ctx = NULL; SFilterColCtx colCtx = {0}; SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); @@ -2258,7 +2198,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); - CHK_JMP(*all); + FLT_CHK_JMP(*all); if (numEqual) { if ((*gRes1)->colNum == 1) { @@ -2278,7 +2218,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++equal2; } - CHK_JMP(equal1 != merNum && equal2 != merNum); + FLT_CHK_JMP(equal1 != merNum && equal2 != merNum); colCtx.colIdx = idx1; colCtx.ctx = ctx; ctx = NULL; @@ -2290,7 +2230,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++equal1; } - CHK_JMP(equal1 != merNum); + FLT_CHK_JMP(equal1 != merNum); colCtx.colIdx = idx1; colCtx.ctx = ctx; ctx = NULL; @@ -2351,7 +2291,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); int32_t pEnd = 0, cStart = 0, cEnd = 0; - uint16_t pColNum = 0, cColNum = 0; + uint32_t pColNum = 0, cColNum = 0; int32_t movedNum = 0; bool all = false; @@ -2371,7 +2311,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR assert(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - CHK_JMP(all); + FLT_CHK_JMP(all); if (gRes[n] == NULL) { if (n < ((*gResNum) - 1)) { @@ -2392,7 +2332,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR assert(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - CHK_JMP(all); + FLT_CHK_JMP(all); if (gRes[n] == NULL) { if (n < ((*gResNum) - 1)) { @@ -2417,7 +2357,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR } cStart = i; - cEnd = i; cColNum = gRes[i]->colNum; } @@ -2433,7 +2372,7 @@ _return: int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { size_t groupSize = taosArrayGetSize(group); - info->groupNum = (uint16_t)groupSize; + info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { info->groups = calloc(info->groupNum, sizeof(*info->groups)); @@ -2450,7 +2389,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { - //qDebug("no need rewrite"); + qDebug("no need rewrite"); return TSDB_CODE_SUCCESS; } @@ -2462,7 +2401,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum SFilterGroupCtx *res = NULL; SFilterColInfo *colInfo = NULL; int32_t optr = 0; - uint16_t uidx = 0; + uint32_t uidx = 0; memset(info, 0, sizeof(*info)); @@ -2482,7 +2421,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum SFilterGroup ng = {0}; - for (uint16_t m = 0; m < res->colNum; ++m) { + for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { assert(colInfo->type == RANGE_TYPE_UNIT); @@ -2518,13 +2457,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum } int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - uint16_t *idxs = NULL; - uint16_t colNum = 0; + uint32_t *idxs = NULL; + uint32_t colNum = 0; SFilterGroupCtx *res = NULL; - uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); for (int32_t i = 0; i < gResNum; ++i) { - for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { + for (uint32_t m = 0; m < gRes[i]->colNum; ++m) { SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { continue; @@ -2534,7 +2473,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ } } - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { if (idxNum[i] < gResNum) { continue; } @@ -2548,16 +2487,16 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ idxs[colNum++] = i; } - CHK_JMP(colNum <= 0); + FLT_CHK_JMP(colNum <= 0); info->colRangeNum = colNum; info->colRange = calloc(colNum, POINTER_BYTES); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - uint16_t n = 0; + uint32_t n = 0; - for (uint16_t m = 0; m < info->colRangeNum; ++m) { + for (uint32_t m = 0; m < info->colRangeNum; ++m) { for (; n < res->colNum; ++n) { if (res->colIdx[n] < idxs[m]) { continue; @@ -2569,7 +2508,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ if (info->colRange[m] == NULL) { info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); - info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; + info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi); } assert(colInfo->type == RANGE_TYPE_MR_CTX); @@ -2588,7 +2527,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ --info->colRangeNum; --m; - CHK_JMP(info->colRangeNum <= 0); + FLT_CHK_JMP(info->colRangeNum <= 0); } ++n; @@ -2605,7 +2544,7 @@ _return: } int32_t filterPostProcessRange(SFilterInfo *info) { - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + for (uint32_t i = 0; i < info->colRangeNum; ++i) { SFilterRangeCtx* ctx = info->colRange[i]; SFilterRangeNode *r = ctx->rs; while (r) { @@ -2619,13 +2558,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) { - uint16_t n = 0; - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); @@ -2635,7 +2572,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant + info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); + }else{ + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } } else { info->cunits[i].valData = NULL; } @@ -2648,30 +2589,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); } - - uint16_t cgroupNum = info->groupNum + 1; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - cgroupNum += info->groups[i].unitNum; - } - - info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); - - for (uint16_t i = 0; i < info->groupNum; ++i) { - info->cgroups[n++] = info->groups[i].unitNum; - - for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { - info->cgroups[n++] = info->groups[i].unitIdxs[m]; - } - } - - info->cgroups[n] = 0; return TSDB_CODE_SUCCESS; } int32_t filterUpdateComUnits(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); @@ -2686,7 +2609,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); - for (int32_t k = 0; k < info->unitNum; ++k) { + for (uint32_t k = 0; k < info->unitNum; ++k) { int32_t index = -1; SFilterComUnit *cunit = &info->cunits[k]; @@ -2731,7 +2654,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } } - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL + if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NOT_EQUAL) { continue; @@ -2739,10 +2662,12 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 SColumnDataAgg* pDataBlockst = &pDataStatis[index]; void *minVal, *maxVal; + float minv = 0; + float maxv = 0; if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); + minv = (float)(*(double *)(&pDataBlockst->min)); + maxv = (float)(*(double *)(&pDataBlockst->max)); minVal = &minv; maxVal = &maxv; @@ -2797,12 +2722,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } -// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); + if (rmUnit == 0) { + fltDebug("NO Block Filter APPLY"); + FLT_RET(TSDB_CODE_SUCCESS); + } info->blkGroupNum = info->groupNum; - uint16_t *unitNum = info->blkUnits; - uint16_t *unitIdx = unitNum + 1; + uint32_t *unitNum = info->blkUnits; + uint32_t *unitIdx = unitNum + 1; int32_t all = 0, empty = 0; for (uint32_t g = 0; g < info->groupNum; ++g) { @@ -2812,7 +2740,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 empty = 0; for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + uint32_t uidx = group->unitIdxs[u]; if (info->blkUnitRes[uidx] == 1) { --(*unitNum); all = 1; @@ -2859,17 +2787,19 @@ _return: bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - uint16_t *unitIdx = NULL; - - *p = calloc(numOfRows, sizeof(int8_t)); + uint32_t *unitIdx = NULL; + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); unitIdx = info->blkUnits; for (uint32_t g = 0; g < info->blkGroupNum; ++g) { - uint16_t unitNum = *(unitIdx++); + uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; void *colData = (char *)cunit->colData + cunit->dataSize * i; @@ -2964,12 +2894,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, return all; } - *p = calloc(numOfRows, sizeof(int8_t)); + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if (!colData){ // for json->'key' is null + (*p)[i] = 1; + }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null + colData = POINTER_SHIFT(colData, CHAR_BYTES); + (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + }else{ + (*p)[i] = 0; + } + }else{ + (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType)); + } if ((*p)[i] == 0) { all = false; } @@ -2985,12 +2928,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } - *p = calloc(numOfRows, sizeof(int8_t)); - + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if (!colData) { // for json->'key' is not null + (*p)[i] = 0; + }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null + colData = POINTER_SHIFT(colData, CHAR_BYTES); + (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + }else{ // for json->'key' is not null + (*p)[i] = 1; + } + }else { + (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType)); + } + if ((*p)[i] == 0) { all = false; } @@ -3013,10 +2971,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD return all; } - *p = calloc(numOfRows, sizeof(int8_t)); + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - if (isNull(colData, info->cunits[0].dataType)) { + if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { all = false; colData += dataSize; continue; @@ -3041,19 +3001,35 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { return all; } - - *p = calloc(numOfRows, sizeof(int8_t)); + + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - if (isNull(colData, info->cunits[uidx].dataType)) { + if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { + (*p)[i] = 0; all = false; continue; } + // match/nmatch for nchar type need convert from ucs4 to mbs + + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){ + char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); + } + tfree(newColData); + }else{ + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + } - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - if ((*p)[i] == 0) { all = false; } @@ -3071,15 +3047,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg return all; } - *p = calloc(numOfRows, sizeof(int8_t)); - + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); for (uint32_t g = 0; g < info->groupNum; ++g) { SFilterGroup *group = &info->groups[g]; for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; void *colData = (char *)cunit->colData + cunit->dataSize * i; @@ -3088,7 +3066,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg //} else { uint8_t optr = cunit->optr; - if (isNull(colData, cunit->dataType)) { + if (colData == NULL || isNull(colData, cunit->dataType)) { (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; } else { if (optr == TSDB_RELATION_NOTNULL) { @@ -3098,7 +3076,19 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){ + char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); + } + tfree(newColData); + }else{ + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } } //FILTER_UNIT_SET_R(info, uidx, p[i]); @@ -3123,11 +3113,6 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg return all; } - -FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { - return (*info->func)(info, numOfRows, p, statis, numOfCols); -} - int32_t filterSetExecFunc(SFilterInfo *info) { if (FILTER_ALL_RES(info)) { info->func = filterExecuteImplAll; @@ -3174,13 +3159,13 @@ int32_t filterPreprocess(SFilterInfo *info) { filterMergeGroups(info, gRes, &gResNum); if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { -// qInfo("Final - FilterInfo: [ALL]"); + fltInfo("Final - FilterInfo: [ALL]"); goto _return; } if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { -// qInfo("Final - FilterInfo: [EMPTY]"); + fltInfo("Final - FilterInfo: [EMPTY]"); goto _return; } @@ -3207,25 +3192,19 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { -// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); -// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); +int32_t fltSetColFieldDataImpl(SFilterInfo *info, void *param, filer_get_col_from_id fp, bool fromColId) { if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { return TSDB_CODE_SUCCESS; } - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema* sch = fi->desc; - - for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); - if (sch->colId == pColInfo->info.colId) { - fi->data = pColInfo->pData; - - break; - } + + if (fromColId) { + (*fp)(param, FILTER_GET_COL_FIELD_ID(fi), &fi->data); + } else { + (*fp)(param, FILTER_GET_COL_FIELD_SLOT_ID(fi), &fi->data); } } @@ -3235,45 +3214,32 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat } -int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { +int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { int32_t code = TSDB_CODE_SUCCESS; - SFilterInfo *info = NULL; - -// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); - - if (*pinfo == NULL) { - *pinfo = calloc(1, sizeof(SFilterInfo)); - } - - info = *pinfo; - - info->options = options; SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); filterInitUnitsFields(info); - code = filterTreeToGroup(tree, info, group); - - ERR_JRET(code); + SFltBuildGroupCtx tctx = {.info = info, .group = group}; + nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); + taosArrayDestroy(group); - ERR_JRET(filterInitValFieldData(info)); + FLT_ERR_JRET(fltInitValFieldData(info)); if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { filterDumpInfoToString(info, "Before preprocess", 0); - ERR_JRET(filterPreprocess(info)); + FLT_ERR_JRET(filterPreprocess(info)); - CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); + FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - taosArrayDestroy(group); return code; } - - //ERR_JRET(filterInitUnitFunc(info)); } info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); @@ -3281,18 +3247,12 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option filterDumpInfoToString(info, "Final", 0); - taosArrayDestroy(group); - return code; _return: -// qInfo("No filter, code:%d", code); + + qInfo("No filter, code:%d", code); - taosArrayDestroy(group); - filterFreeInfo(*pinfo); - - *pinfo = NULL; - return code; } @@ -3311,7 +3271,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t bool ret = true; void *minVal, *maxVal; - for (int32_t k = 0; k < info->colRangeNum; ++k) { + for (uint32_t k = 0; k < info->colRangeNum; ++k) { int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for(int32_t i = 0; i < numOfCols; ++i) { @@ -3331,30 +3291,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t break; } - if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { - ret = false; - break; - } - - // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { - - // if isNULL query exists, load the null data column - if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + if (pDataStatis[index].numOfNull <= 0) { + if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } + } else if (pDataStatis[index].numOfNull > 0) { + if (pDataStatis[index].numOfNull == numOfRows) { + if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + ret = false; + break; + } - continue; + continue; + } else { + if (ctx->isnull) { + continue; + } + } } SColumnDataAgg* pDataBlockst = &pDataStatis[index]; SFilterRangeNode *r = ctx->rs; + float minv = 0; + float maxv = 0; if (ctx->type == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); + minv = (float)(*(double *)(&pDataBlockst->min)); + maxv = (float)(*(double *)(&pDataBlockst->max)); minVal = &minv; maxVal = &maxv; @@ -3371,7 +3336,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t r = r->next; } - CHK_RET(!ret, ret); + if (!ret) { + return ret; + } } return ret; @@ -3389,7 +3356,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { int32_t code = 0; bool empty = false, all = false; - for (int32_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup *group = &info->groups[i]; if (group->unitNum > 1) { cur = tmpc; @@ -3399,14 +3366,14 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { optr = TSDB_RELATION_OR; } - for (int32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint32_t uidx = group->unitIdxs[u]; SFilterUnit *unit = &info->units[uidx]; uint8_t raOptr = FILTER_UNIT_OPTR(unit); filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); - CHK_JMP(empty); + FLT_CHK_JMP(empty); if (FILTER_NO_MERGE_OPTR(raOptr)) { continue; @@ -3441,11 +3408,11 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { } else { filterGetRangeNum(prev, &num); if (num > 1) { - //qError("only one time range accepted, num:%d", num); - ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); + qError("only one time range accepted, num:%d", num); + FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); } - CHK_JMP(num < 1); + FLT_CHK_JMP(num < 1); SFilterRange tra; filterGetRangeRes(prev, &tra); @@ -3456,7 +3423,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { filterFreeRangeCtx(prev); filterFreeRangeCtx(tmpc); - //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); return TSDB_CODE_SUCCESS; _return: @@ -3466,14 +3433,18 @@ _return: filterFreeRangeCtx(prev); filterFreeRangeCtx(tmpc); - //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); return code; } int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { @@ -3486,7 +3457,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar char *src = FILTER_GET_COL_FIELD_DATA(fi, j); char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + char *varSrc = varDataVal(src); + size_t k = 0, varSrcLen = varDataLen(src); + while (k < varSrcLen && varSrc[k++] == -1) {} + if (k == varSrcLen) { + /* NULL */ + varDataLen(dst) = (VarDataLenT) varSrcLen; + varDataCopy(dst, src); + continue; + } + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + if(!ret) { + qError("filterConverNcharColumns taosMbsToUcs4 error"); + return TSDB_CODE_FAILED; + } varDataLen(dst) = len; } @@ -3504,7 +3488,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar } int32_t filterFreeNcharColumns(SFilterInfo* info) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { @@ -3515,7 +3499,198 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { return TSDB_CODE_SUCCESS; } +EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { + SFltTreeStat *stat = (SFltTreeStat *)pContext; + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + fltError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + stat->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + if ((QUERY_NODE_OPERATOR != nodeType(cell->pNode)) && (QUERY_NODE_LOGIC_CONDITION != nodeType(cell->pNode))) { + stat->scalarMode = true; + } + + cell = cell->pNext; + } + + return DEAL_RES_CONTINUE; + } + + if (stat->scalarMode) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + SOperatorNode *node = (SOperatorNode *)*pNode; + if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (NULL == node->pRight) { + if (scalarGetOperatorParamNum(node->opType) > 1) { + fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node)); + stat->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } else { + if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (nodeType(node->pLeft) == nodeType(node->pRight)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + SNode *t = node->pLeft; + node->pLeft = node->pRight; + node->pRight = t; + } + + if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { + fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); + stat->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + if (OP_TYPE_IN != node->opType) { + SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SValueNode *valueNode = (SValueNode *)node->pRight; + int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->dataType.type) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } + } + + return DEAL_RES_CONTINUE; + } + + fltError("invalid node type for filter, type:%d", nodeType(*pNode)); + + stat->code = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; +} + +int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + + FLT_RET(pStat->code); +} + +int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + +} + + +int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { + int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; + SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); + if (id == pColInfo->info.colId) { + *data = pColInfo->pData; + break; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + return fltSetColFieldDataImpl(info, param, fp, false); +} + +int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + return fltSetColFieldDataImpl(info, param, fp, true); +} +int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) { + int32_t code = 0; + SFilterInfo *info = NULL; + + if (pNode == NULL || pInfo == NULL) { + fltError("invalid param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (*pInfo == NULL) { + *pInfo = calloc(1, sizeof(SFilterInfo)); + if (NULL == *pInfo) { + fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + info = *pInfo; + info->options = options; + + SFltTreeStat stat = {0}; + FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + + if (!info->scalarMode) { + FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); + } else { + info->sclCtx.node = pNode; + FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat)); + } + + return code; + +_return: + + filterFreeInfo(*pInfo); + + *pInfo = NULL; + + FLT_RET(code); +} + +FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + if (info->scalarMode) { + SScalarParam output = {0}; + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); + + *p = output.data; + return TSDB_CODE_SUCCESS; + } + + return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); +} + + diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c new file mode 100644 index 0000000000..aa29b02709 --- /dev/null +++ b/source/libs/scalar/src/scalar.c @@ -0,0 +1,594 @@ +#include "nodes.h" +#include "common.h" +#include "querynodes.h" +#include "function.h" +#include "functionMgt.h" +#include "sclvector.h" +#include "sclInt.h" + +int32_t scalarGetOperatorParamNum(EOperatorType type) { + if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) { + return 1; + } + + return 2; +} + +void sclFreeRes(SHashObj *res) { + SScalarParam *p = NULL; + void *pIter = taosHashIterate(res, NULL); + while (pIter) { + p = (SScalarParam *)pIter; + + if (p) { + tfree(p->data); + } + + pIter = taosHashIterate(res, pIter); + } + + taosHashCleanup(res); +} + +void sclFreeParam(SScalarParam *param) { + tfree(param->data); +} + +int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { + switch (nodeType(node)) { + case QUERY_NODE_VALUE: { + SValueNode *valueNode = (SValueNode *)node; + param->data = nodesGetValueFromNode(valueNode); + param->num = 1; + param->type = valueNode->node.resType.type; + param->bytes = valueNode->node.resType.bytes; + + break; + } + case QUERY_NODE_NODE_LIST: { + SNodeListNode *nodeList = (SNodeListNode *)node; + //TODO BUILD HASH + break; + } + case QUERY_NODE_COLUMN_REF: { + if (NULL == ctx) { + sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SColumnRefNode *ref = (SColumnRefNode *)node; + if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { + sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); + param->data = columnData->pData; + param->num = ctx->pSrc->info.rows; + param->type = columnData->info.type; + param->bytes = columnData->info.bytes; + + break; + } + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_OPERATOR: { + if (NULL == ctx) { + sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); + if (NULL == res) { + sclError("no result for node, type:%d, node:%p", nodeType(node), node); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *param = *res; + + break; + } + } + + if (param->num > *rowNum) { + if ((1 != param->num) && (1 < *rowNum)) { + sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->num); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + *rowNum = param->num; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { + SScalarParam *param = NULL; + + for (int32_t i = 0; i < num; ++i) { + param = params + i; + + if (1 == param->num) { + continue; + } + + if (IS_VAR_DATA_TYPE(param->type)) { + param->data = (char *)(param->data) + varDataTLen(param->data); + } else { + param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { + int32_t code = 0; + SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == paramList) { + sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SListCell *cell = pParamList->pHead; + for (int32_t i = 0; i < pParamList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SCL_ERR_JRET(sclInitParam(cell->pNode, ¶mList[i], ctx, rowNum)); + + cell = cell->pNext; + } + + *pParams = paramList; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(paramList); + SCL_RET(code); +} + +int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) { + int32_t code = 0; + int32_t paramNum = scalarGetOperatorParamNum(node->opType); + if (NULL == node->pLeft || (paramNum == 2 && NULL == node->pRight)) { + sclError("invalid operation node, left:%p, right:%p", node->pLeft, node->pRight); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam)); + if (NULL == paramList) { + sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); + if (paramNum > 1) { + SCL_ERR_JRET(sclInitParam(node->pRight, ¶mList[1], ctx, rowNum)); + } + + *pParams = paramList; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(paramList); + SCL_RET(code); +} + + +int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarFuncExecFuncs ffpSet = {0}; + int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); + if (code) { + sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_RET(code); + } + + SScalarParam *params = NULL; + int32_t rowNum = 0; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < rowNum; ++i) { + code = (*ffpSet.process)(params, node->pParameterList->length, output); + if (code) { + sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } + + sclParamMoveNext(output, 1); + sclParamMoveNext(params, node->pParameterList->length); + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + + +int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *output) { + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (TSDB_DATA_TYPE_BOOL != node->node.resType.type) { + sclError("invalid logic resType, type:%d", node->node.resType.type); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (LOGIC_COND_TYPE_NOT == node->condType && node->pParameterList->length > 1) { + sclError("invalid NOT operation parameter number, paramNum:%d", node->pParameterList->length); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarParam *params = NULL; + int32_t rowNum = 0; + int32_t code = 0; + + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, sizeof(bool)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + bool value = false; + + for (int32_t i = 0; i < rowNum; ++i) { + for (int32_t m = 0; m < node->pParameterList->length; ++m) { + GET_TYPED_DATA(value, bool, params[m].type, params[m].data); + + if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + break; + } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + break; + } else if (LOGIC_COND_TYPE_NOT == node->condType) { + value = !value; + } + } + + *(bool *)output->data = value; + + sclParamMoveNext(output, 1); + sclParamMoveNext(params, node->pParameterList->length); + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + +int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *output) { + SScalarParam *params = NULL; + int32_t rowNum = 0; + int32_t code = 0; + + SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, tDataTypes[output->type].bytes); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); + + int32_t paramNum = scalarGetOperatorParamNum(node->opType); + SScalarParam* pLeft = ¶ms[0]; + SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; + + for (int32_t i = 0; i < rowNum; ++i) { + + OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); + + sclParamMoveNext(output, 1); + sclParamMoveNext(pLeft, 1); + if (pRight) { + sclParamMoveNext(pRight, 1); + } + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + + +EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { + SFunctionNode *node = (SFunctionNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecFuncion(node, NULL, &output); + if (*(int32_t *)pContext) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + + return DEAL_RES_CONTINUE; +} + +EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecLogic(node, NULL, &output); + if (*(int32_t *)pContext) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + + return DEAL_RES_CONTINUE; +} + +EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { + SOperatorNode *node = (SOperatorNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecOperator(node, NULL, &output); + if (*(int32_t *)pContext) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + return sclRewriteFunction(pNode, pContext); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + return sclRewriteLogic(pNode, pContext); + } + + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclRewriteOperator(pNode, pContext); + } + + sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); + + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; +} + + +EDealRes sclWalkFunction(SNode* pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SFunctionNode *node = (SFunctionNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecFuncion(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclWalkLogic(SNode* pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SLogicConditionNode *node = (SLogicConditionNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecLogic(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclWalkOperator(SNode* pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SOperatorNode *node = (SOperatorNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecOperator(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclCalcWalker(SNode* pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + return sclWalkFunction(pNode, pContext); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + return sclWalkLogic(pNode, pContext); + } + + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + return sclWalkOperator(pNode, pContext); + } + + sclError("invalid node type for calculating constants, type:%d", nodeType(pNode)); + + SScalarCtx *ctx = (SScalarCtx *)pContext; + + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; +} + + + +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { + if (NULL == pNode) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + + nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code); + + if (code) { + nodesDestroyNode(pNode); + SCL_ERR_RET(code); + } + + *pRes = pNode; + + SCL_RET(code); +} + +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { + if (NULL == pNode || NULL == pSrc || NULL == pDst) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + SScalarCtx ctx = {.code = 0, .pSrc = pSrc}; + + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); + + if (ctx.code) { + nodesDestroyNode(pNode); + sclFreeRes(ctx.pRes); + SCL_ERR_RET(ctx.code); + } + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (NULL == res) { + sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode)); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *pDst = *res; + + nodesDestroyNode(pNode); + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/scalar/src/sclfunc.c similarity index 91% rename from source/libs/function/src/tscalarfunction.c rename to source/libs/scalar/src/sclfunc.c index 90d25b3e4a..ab2cc8b056 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1,14 +1,13 @@ -#include "tscalarfunction.h" -#include "tbinoperator.h" -#include "tunaryoperator.h" +#include "sclfunc.h" +#include "sclvector.h" -static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { +static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { dst->type = src->type; dst->bytes = src->bytes; dst->num = src->num; } -static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tceil(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -34,7 +33,7 @@ static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun } } -static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tfloor(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -62,7 +61,7 @@ static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu } } -static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void _tabs(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -120,7 +119,7 @@ static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun } } -static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tround(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -146,7 +145,7 @@ static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu } } -static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assert(numOfInput == 1); int64_t* out = (int64_t*) pOutput->data; @@ -157,7 +156,7 @@ static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF } } -static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assert(numOfInput > 0); int32_t rowLen = 0; @@ -189,11 +188,11 @@ static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF } } -static void tltrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { } -static void trtrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { } @@ -262,7 +261,7 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -static void setScalarFuncParam(SScalarFuncParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { +static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { param->bytes = bytes; param->type = type; param->num = numOfRows; @@ -273,6 +272,7 @@ bool isStringOp(int32_t op) { return op == TSDB_BINARY_OP_CONCAT; } +#if 0 int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)) { if (pExprs == NULL) { @@ -361,6 +361,8 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa return 0; } +#endif + SScalarFunctionInfo scalarFunc[8] = { {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c new file mode 100644 index 0000000000..5b2ec7902c --- /dev/null +++ b/source/libs/scalar/src/sclvector.c @@ -0,0 +1,1177 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "ttypes.h" +#include "sclvector.h" +#include "tcompare.h" +#include "querynodes.h" +#include "filterInt.h" +#include "query.h" + +//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); + +void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { + int32_t *pLeft = (int32_t *)left; + int32_t *pRight = (int32_t *)right; + double * pOutput = (double *)output; + + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; + + if (numLeft == numRight) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[i] + pRight[i]; + } + } else if (numLeft == 1) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[0] + pRight[i]; + } + } else if (numRight == 1) { + for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + *pOutput = (double)pLeft[i] + pRight[0]; + } + } +} + +typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); + +double getVectorDoubleValue_TINYINT(void *src, int32_t index) { + return (double)*((int8_t *)src + index); +} +double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { + return (double)*((uint8_t *)src + index); +} +double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { + return (double)*((int16_t *)src + index); +} +double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { + return (double)*((uint16_t *)src + index); +} +double getVectorDoubleValue_INT(void *src, int32_t index) { + return (double)*((int32_t *)src + index); +} +double getVectorDoubleValue_UINT(void *src, int32_t index) { + return (double)*((uint32_t *)src + index); +} +double getVectorDoubleValue_BIGINT(void *src, int32_t index) { + return (double)*((int64_t *)src + index); +} +double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { + return (double)*((uint64_t *)src + index); +} +double getVectorDoubleValue_FLOAT(void *src, int32_t index) { + return (double)*((float *)src + index); +} +double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { + return (double)*((double *)src + index); +} +_getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { + _getDoubleValue_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorDoubleValue_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorDoubleValue_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorDoubleValue_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorDoubleValue_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorDoubleValue_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorDoubleValue_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorDoubleValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorDoubleValue_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorDoubleValue_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorDoubleValue_DOUBLE; + }else { + assert(0); + } + return p; +} + + + +typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); + +int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { + return (int64_t)*((int8_t *)src + index); +} +int64_t getVectorBigintValue_UTINYINT(void *src, int32_t index) { + return (int64_t)*((uint8_t *)src + index); +} +int64_t getVectorBigintValue_SMALLINT(void *src, int32_t index) { + return (int64_t)*((int16_t *)src + index); +} +int64_t getVectorBigintValue_USMALLINT(void *src, int32_t index) { + return (int64_t)*((uint16_t *)src + index); +} +int64_t getVectorBigintValue_INT(void *src, int32_t index) { + return (int64_t)*((int32_t *)src + index); +} +int64_t getVectorBigintValue_UINT(void *src, int32_t index) { + return (int64_t)*((uint32_t *)src + index); +} +int64_t getVectorBigintValue_BIGINT(void *src, int32_t index) { + return (int64_t)*((int64_t *)src + index); +} +int64_t getVectorBigintValue_UBIGINT(void *src, int32_t index) { + return (int64_t)*((uint64_t *)src + index); +} +int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) { + return (int64_t)*((float *)src + index); +} +int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) { + return (int64_t)*((double *)src + index); +} +_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { + _getBigintValue_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorBigintValue_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorBigintValue_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorBigintValue_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorBigintValue_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorBigintValue_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorBigintValue_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorBigintValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorBigintValue_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorBigintValue_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorBigintValue_DOUBLE; + }else { + assert(0); + } + return p; +} + + + + +typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index); + +void* getVectorValueAddr_TINYINT(void *src, int32_t index) { + return (void*)((int8_t *)src + index); +} +void* getVectorValueAddr_UTINYINT(void *src, int32_t index) { + return (void*)((uint8_t *)src + index); +} +void* getVectorValueAddr_SMALLINT(void *src, int32_t index) { + return (void*)((int16_t *)src + index); +} +void* getVectorValueAddr_USMALLINT(void *src, int32_t index) { + return (void*)((uint16_t *)src + index); +} +void* getVectorValueAddr_INT(void *src, int32_t index) { + return (void*)((int32_t *)src + index); +} +void* getVectorValueAddr_UINT(void *src, int32_t index) { + return (void*)((uint32_t *)src + index); +} +void* getVectorValueAddr_BIGINT(void *src, int32_t index) { + return (void*)((int64_t *)src + index); +} +void* getVectorValueAddr_UBIGINT(void *src, int32_t index) { + return (void*)((uint64_t *)src + index); +} +void* getVectorValueAddr_FLOAT(void *src, int32_t index) { + return (void*)((float *)src + index); +} +void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { + return (void*)((double *)src + index); +} + +_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { + _getValueAddr_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorValueAddr_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorValueAddr_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorValueAddr_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorValueAddr_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorValueAddr_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorValueAddr_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorValueAddr_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorValueAddr_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorValueAddr_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorValueAddr_DOUBLE; + }else { + assert(0); + } + return p; +} + + +int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { + int16_t inType = pIn->type; + int16_t inBytes = pIn->bytes; + char *input = pIn->data; + int16_t outType = pOut->type; + int16_t outBytes = pOut->bytes; + char *output = pOut->data; + + switch (outType) { + case TSDB_DATA_TYPE_BOOL: + if (inType == TSDB_DATA_TYPE_BINARY) { + for (int32_t i = 0; i < pIn->num; ++i) { + GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += sizeof(bool); + } + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + for (int32_t i = 0; i < pIn->num; ++i) { + GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + qError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + qError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + qError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + default: + qError("invalid convert output type:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { +/* NULL BOOL TINY SMAL INT BIG FLOA DOUB BINA TIME NCHA UTIN USMA UINT UBIG VARC VARB JSON DECI BLOB */ +/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 7, 0, 0, 0, +/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 7, 0, 0, 0, +/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 7, 0, 0, 0, +/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 7, 0, 0, 0, +/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 7, 0, 0, 0, +/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 7, 0, 0, 0, +/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, +/*BINA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, +/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 7, 0, 0, 0, +/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, +/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 7, 0, 0, 0, +/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 7, 0, 0, 0, +/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 7, 0, 0, 0, +/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +int32_t vectorGetConvertType(int32_t type1, int32_t type2) { + if (type1 == type2) { + return 0; + } + + if (type1 < type2) { + return gConvertTypes[type1][type2]; + } + + return gConvertTypes[type2][type1]; +} + +int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) { + if (pLeft->type == pRight->type) { + return TSDB_CODE_SUCCESS; + } + + SScalarParam *param1 = NULL, *paramOut1 = NULL; + SScalarParam *param2 = NULL, *paramOut2 = NULL; + int32_t code = 0; + + if (pLeft->type < pRight->type) { + param1 = pLeft; + param2 = pRight; + paramOut1 = pLeftOut; + paramOut2 = pRightOut; + } else { + param1 = pRight; + param2 = pLeft; + paramOut1 = pRightOut; + paramOut2 = pLeftOut; + } + + + int8_t type = vectorGetConvertType(param1->type, param2->type); + if (0 == type) { + return TSDB_CODE_SUCCESS; + } + + if (type != param1->type) { + paramOut1->bytes = param1->bytes; + paramOut1->type = type; + paramOut1->num = param1->num; + paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); + if (NULL == paramOut1->data) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + code = vectorConvertImpl(param1, paramOut1); + if (code) { + tfree(paramOut1->data); + return code; + } + } + + if (type != param2->type) { + paramOut2->bytes = param2->bytes; + paramOut2->type = type; + paramOut2->num = param2->num; + paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); + if (NULL == paramOut2->data) { + tfree(paramOut1->data); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + code = vectorConvertImpl(param2, paramOut2); + if (code) { + tfree(paramOut1->data); + tfree(paramOut2->data); + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + } + } +} + +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); + } + } +} +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); + } + } +} + +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + double right = getVectorDoubleValueFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(&right, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); + } + } +} + +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double * output = (double *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double left = getVectorDoubleValueFnLeft(pLeft->data, i); + double right = getVectorDoubleValueFnRight(pRight->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } else if (pLeft->num == 1) { + double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(&left, pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double right = getVectorDoubleValueFnRight(pRight->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } else if (pRight->num == 1) { + double right = getVectorDoubleValueFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(&right, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double left = getVectorDoubleValueFnLeft(pLeft->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } +} + +void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t len = pLeft->bytes + pRight->bytes; + + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + char *output = (char *)out; + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += len) { + char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); + char* right = POINTER_SHIFT(pRight->data, pRight->bytes * i); + + if (isNull(left, pLeft->type) || isNull(right, pRight->type)) { + setVardataNull(output, TSDB_DATA_TYPE_BINARY); + continue; + } + + // todo define a macro + memcpy(varDataVal(output), varDataVal(left), varDataLen(left)); + memcpy(varDataVal(output) + varDataLen(left), varDataVal(right), varDataLen(right)); + varDataSetLen(output, varDataLen(left) + varDataLen(right)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += len) { + char *right = POINTER_SHIFT(pRight->data, pRight->bytes * i); + if (isNull(pLeft->data, pLeft->type) || isNull(right, pRight->type)) { + setVardataNull(output, TSDB_DATA_TYPE_BINARY); + continue; + } + + memcpy(varDataVal(output), varDataVal(pLeft->data), varDataLen(pLeft->data)); + memcpy(varDataVal(output) + varDataLen(pLeft->data), varDataVal(right), varDataLen(right)); + varDataSetLen(output, varDataLen(pLeft->data) + varDataLen(right)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += len) { + char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); + if (isNull(left, pLeft->type) || isNull(pRight->data, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + memcpy(varDataVal(output), varDataVal(left), varDataLen(pRight->data)); + memcpy(varDataVal(output) + varDataLen(left), varDataVal(pRight->data), varDataLen(pRight->data)); + varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); + } + } + +} + + +void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + int64_t *output=(int64_t *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + + SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + } + } +} + +void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + int64_t *output=(int64_t *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + + SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + } + } +} + + +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr); + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i)); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } else if (pLeft->num == 1) { + void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i)); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } else if (pRight->num == 1) { + void *rightData = getVectorValueAddrFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } +} + +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { + SScalarParam pLeftOut = {0}; + SScalarParam pRightOut = {0}; + + vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); + + SScalarParam *param1 = NULL; + SScalarParam *param2 = NULL; + + int32_t type = 0; + if (pLeftOut.type) { + param1 = &pLeftOut; + } else { + param1 = pLeft; + } + + if (pRightOut.type) { + param2 = &pRightOut; + } else { + param2 = pRight; + } + + vectorCompareImpl(param1, param2, out, _ord, TSDB_RELATION_GREATER); +} + +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER); +} + +void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL); +} + +void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS); +} + +void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL); +} + +void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL); +} + +void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL); +} + +void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN); +} + +void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN); +} + +void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE); +} + +void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE); +} + +void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH); +} + +void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH); +} + +void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + res = true; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } +} + +void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + bool res = false; + + bool *output = (bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = true; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } +} + + +_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { + switch (binFunctionId) { + case OP_TYPE_ADD: + return vectorAdd; + case OP_TYPE_SUB: + return vectorSub; + case OP_TYPE_MULTI: + return vectorMultiply; + case OP_TYPE_DIV: + return vectorDivide; + case OP_TYPE_MOD: + return vectorRemainder; + case OP_TYPE_GREATER_THAN: + return vectorGreater; + case OP_TYPE_GREATER_EQUAL: + return vectorGreaterEqual; + case OP_TYPE_LOWER_THAN: + return vectorLower; + case OP_TYPE_LOWER_EQUAL: + return vectorLowerEqual; + case OP_TYPE_EQUAL: + return vectorEqual; + case OP_TYPE_NOT_EQUAL: + return vectorNotEqual; + case OP_TYPE_IN: + return vectorIn; + case OP_TYPE_NOT_IN: + return vectorNotIn; + case OP_TYPE_LIKE: + return vectorLike; + case OP_TYPE_NOT_LIKE: + return vectorNotLike; + case OP_TYPE_MATCH: + return vectorMatch; + case OP_TYPE_NMATCH: + return vectorNotMatch; + case OP_TYPE_IS_NULL: + return vectorIsNull; + case OP_TYPE_IS_NOT_NULL: + return vectorNotNull; + case OP_TYPE_BIT_AND: + return vectorBitAnd; + case OP_TYPE_BIT_OR: + return vectorBitOr; + default: + assert(0); + return NULL; + } +} + +bool isBinaryStringOp(int32_t op) { + return op == TSDB_BINARY_OP_CONCAT; +} diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt new file mode 100644 index 0000000000..a9af1ece30 --- /dev/null +++ b/source/libs/scalar/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scalar unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + scalarTest + PUBLIC os util common gtest qcom function nodes +) + +TARGET_INCLUDE_DIRECTORIES( + scalarTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/scalarTests.cpp b/source/libs/scalar/test/scalarTests.cpp new file mode 100644 index 0000000000..38fe072f9e --- /dev/null +++ b/source/libs/scalar/test/scalarTests.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" + +namespace { + +} + +TEST(scalarTest, func) { + +} + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 784a864451..37ee5194c8 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -1,4 +1,3 @@ - aux_source_directory(src SYNC_SRC) add_library(sync ${SYNC_SRC}) diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h new file mode 100644 index 0000000000..9ca0de19c5 --- /dev/null +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_APPEND_ENTRIES_H +#define _TD_LIBS_SYNC_APPEND_ENTRIES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg); + +void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_APPEND_ENTRIES_H*/ diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h new file mode 100644 index 0000000000..8b5cbf1da5 --- /dev/null +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H +#define _TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H*/ diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h new file mode 100644 index 0000000000..34dfdb3d09 --- /dev/null +++ b/source/libs/sync/inc/syncElection.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_ELECTION_H +#define _TD_LIBS_SYNC_ELECTION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_ELECTION_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h new file mode 100644 index 0000000000..551ce83122 --- /dev/null +++ b/source/libs/sync/inc/syncInt.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_INT_H +#define _TD_LIBS_SYNC_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + +#define sFatal(...) \ + { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sError(...) \ + { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sWarn(...) \ + { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sInfo(...) \ + { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sDebug(...) \ + { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sTrace(...) \ + { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ + } \ + } + +typedef struct SSyncNode { + char path[TSDB_FILENAME_LEN]; + int8_t replica; + int8_t quorum; + int8_t selfIndex; + uint32_t vgId; + int32_t refCount; + int64_t rid; +} SSyncNode; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_INT_H*/ diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h new file mode 100644 index 0000000000..f410c8cf6e --- /dev/null +++ b/source/libs/sync/inc/syncMessage.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_MESSAGE_H +#define _TD_LIBS_SYNC_MESSAGE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncRaftEntry.h" +#include "taosdef.h" + +typedef enum ESyncMessageType { + SYNC_PING = 0, + SYNC_PING_REPLY, + SYNC_CLIENT_REQUEST, + SYNC_CLIENT_REQUEST_REPLY, + SYNC_REQUEST_VOTE, + SYNC_REQUEST_VOTE_REPLY, + SYNC_APPEND_ENTRIES, + SYNC_APPEND_ENTRIES_REPLY, +} ESyncMessageType; + +typedef struct SyncPing { + ESyncMessageType msgType; + const SSyncBuffer *pData; +} SyncPing; + +typedef struct SyncPingReply { + ESyncMessageType msgType; + const SSyncBuffer *pData; +} SyncPingReply; + +typedef struct SyncClientRequest { + ESyncMessageType msgType; + const SSyncBuffer *pData; + int64_t seqNum; + bool isWeak; +} SyncClientRequest; + +typedef struct SyncClientRequestReply { + ESyncMessageType msgType; + int32_t errCode; + const SSyncBuffer *pErrMsg; + const SSyncBuffer *pLeaderHint; +} SyncClientRequestReply; + +typedef struct SyncRequestVote { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + SyncGroupId vgId; + SyncIndex lastLogIndex; + SyncTerm lastLogTerm; +} SyncRequestVote; + +typedef struct SyncRequestVoteReply { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + SyncGroupId vgId; + bool voteGranted; +} SyncRequestVoteReply; + +typedef struct SyncAppendEntries { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + SyncIndex prevLogIndex; + SyncTerm prevLogTerm; + int32_t entryCount; + SSyncRaftEntry * logEntries; + SyncIndex commitIndex; +} SyncAppendEntries; + +typedef struct SyncAppendEntriesReply { + ESyncMessageType msgType; + SyncTerm currentTerm; + SyncNodeId nodeId; + bool success; + SyncIndex matchIndex; +} SyncAppendEntriesReply; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_MESSAGE_H*/ diff --git a/source/libs/sync/inc/syncOnMessage.h b/source/libs/sync/inc/syncOnMessage.h new file mode 100644 index 0000000000..07c44b6199 --- /dev/null +++ b/source/libs/sync/inc/syncOnMessage.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_ON_MESSAGE_H +#define _TD_LIBS_SYNC_ON_MESSAGE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncRaft.h" +#include "taosdef.h" + +void onMessage(SRaft *pRaft, void *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_ON_MESSAGE_H*/ diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h new file mode 100644 index 0000000000..0c7e573572 --- /dev/null +++ b/source/libs/sync/inc/syncRaft.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_H +#define _TD_LIBS_SYNC_RAFT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncMessage.h" +#include "taosdef.h" + +typedef struct SRaftId { + SyncNodeId nodeId; + SyncGroupId vgId; +} SRaftId; + +typedef struct SRaft { + SRaftId id; +} SRaft; + +int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); + +static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_H*/ diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h new file mode 100644 index 0000000000..adc82f2c5d --- /dev/null +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_TPL_H +#define _TD_LIBS_TPL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "taosdef.h" + +typedef struct SSyncRaftEntry { + SyncTerm term; + SyncIndex index; + SSyncBuffer data; + int8_t flag; +} SSyncRaftEntry; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_TPL_H*/ diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h new file mode 100644 index 0000000000..8c4b5116ea --- /dev/null +++ b/source/libs/sync/inc/syncRaftLog.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_LOG_H +#define _TD_LIBS_SYNC_RAFT_LOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "taosdef.h" + +int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); + +// get one log entry, user need to free pBuf->data +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); + +// update log store commit index with "index" +int32_t raftLogUpdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index); + +// truncate log with index, entries after the given index (>index) will be deleted +int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index); + +// return commit index of log +SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore); + +// return index of last entry +SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore); + +// return term of last entry +SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_LOG_H*/ diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h new file mode 100644 index 0000000000..4cb852f34a --- /dev/null +++ b/source/libs/sync/inc/syncRaftStore.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_STORE_H +#define _TD_LIBS_SYNC_RAFT_STORE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncRaft.h" +#include "taosdef.h" + +int32_t currentTerm(SyncTerm *pCurrentTerm); + +int32_t persistCurrentTerm(SyncTerm currentTerm); + +int32_t voteFor(SRaftId *pRaftId); + +int32_t persistVoteFor(SRaftId *pRaftId); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_STORE_H*/ diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h new file mode 100644 index 0000000000..40c5ff790b --- /dev/null +++ b/source/libs/sync/inc/syncReplication.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_REPLICATION_H +#define _TD_LIBS_SYNC_REPLICATION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_REPLICATION_H*/ diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h new file mode 100644 index 0000000000..3ff96bbe8f --- /dev/null +++ b/source/libs/sync/inc/syncRequestVote.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_REQUEST_VOTE_H +#define _TD_LIBS_SYNC_REQUEST_VOTE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg); + +void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_REQUEST_VOTE_H*/ diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h new file mode 100644 index 0000000000..033ac89bc2 --- /dev/null +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H +#define _TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H*/ diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h new file mode 100644 index 0000000000..3b6121578a --- /dev/null +++ b/source/libs/sync/inc/syncSnapshot.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_SNAPSHOT_H +#define _TD_LIBS_SYNC_SNAPSHOT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "sync.h" +#include "syncRaft.h" +#include "taosdef.h" + +int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); + +int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_SNAPSHOT_H*/ diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h new file mode 100644 index 0000000000..8159d2566c --- /dev/null +++ b/source/libs/sync/inc/syncTimeout.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_TIMEOUT_H +#define _TD_LIBS_SYNC_TIMEOUT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncMessage.h" +#include "syncRaft.h" +#include "taosdef.h" + +void onTimeout(SRaft *pRaft, void *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_TIMEOUT_H*/ diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c deleted file mode 100644 index 7ded53b6e6..0000000000 --- a/source/libs/sync/src/sync.c +++ /dev/null @@ -1 +0,0 @@ -#include "sync.h" \ No newline at end of file diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c new file mode 100644 index 0000000000..1286108664 --- /dev/null +++ b/source/libs/sync/src/syncAppendEntries.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncAppendEntries.h" +#include "sync.h" + +void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { + +// TLA+ Spec +//AppendEntries(i, j) == +// /\ i /= j +// /\ state[i] = Leader +// /\ LET prevLogIndex == nextIndex[i][j] - 1 +// prevLogTerm == IF prevLogIndex > 0 THEN +// log[i][prevLogIndex].term +// ELSE +// 0 +// \* Send up to 1 entry, constrained by the end of the log. +// lastEntry == Min({Len(log[i]), nextIndex[i][j]}) +// entries == SubSeq(log[i], nextIndex[i][j], lastEntry) +// IN Send([mtype |-> AppendEntriesRequest, +// mterm |-> currentTerm[i], +// mprevLogIndex |-> prevLogIndex, +// mprevLogTerm |-> prevLogTerm, +// mentries |-> entries, +// \* mlog is used as a history variable for the proof. +// \* It would not exist in a real implementation. +// mlog |-> log[i], +// mcommitIndex |-> Min({commitIndex[i], lastEntry}), +// msource |-> i, +// mdest |-> j]) +// /\ UNCHANGED <> + +} + +void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { + +// TLA+ Spec +//HandleAppendEntriesRequest(i, j, m) == +// LET logOk == \/ m.mprevLogIndex = 0 +// \/ /\ m.mprevLogIndex > 0 +// /\ m.mprevLogIndex <= Len(log[i]) +// /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term +// IN /\ m.mterm <= currentTerm[i] +// /\ \/ /\ \* reject request +// \/ m.mterm < currentTerm[i] +// \/ /\ m.mterm = currentTerm[i] +// /\ state[i] = Follower +// /\ \lnot logOk +// /\ Reply([mtype |-> AppendEntriesResponse, +// mterm |-> currentTerm[i], +// msuccess |-> FALSE, +// mmatchIndex |-> 0, +// msource |-> i, +// mdest |-> j], +// m) +// /\ UNCHANGED <> +// \/ \* return to follower state +// /\ m.mterm = currentTerm[i] +// /\ state[i] = Candidate +// /\ state' = [state EXCEPT ![i] = Follower] +// /\ UNCHANGED <> +// \/ \* accept request +// /\ m.mterm = currentTerm[i] +// /\ state[i] = Follower +// /\ logOk +// /\ LET index == m.mprevLogIndex + 1 +// IN \/ \* already done with request +// /\ \/ m.mentries = << >> +// \/ /\ m.mentries /= << >> +// /\ Len(log[i]) >= index +// /\ log[i][index].term = m.mentries[1].term +// \* This could make our commitIndex decrease (for +// \* example if we process an old, duplicated request), +// \* but that doesn't really affect anything. +// /\ commitIndex' = [commitIndex EXCEPT ![i] = +// m.mcommitIndex] +// /\ Reply([mtype |-> AppendEntriesResponse, +// mterm |-> currentTerm[i], +// msuccess |-> TRUE, +// mmatchIndex |-> m.mprevLogIndex + +// Len(m.mentries), +// msource |-> i, +// mdest |-> j], +// m) +// /\ UNCHANGED <> +// \/ \* conflict: remove 1 entry +// /\ m.mentries /= << >> +// /\ Len(log[i]) >= index +// /\ log[i][index].term /= m.mentries[1].term +// /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |-> +// log[i][index2]] +// IN log' = [log EXCEPT ![i] = new] +// /\ UNCHANGED <> +// \/ \* no conflict: append entry +// /\ m.mentries /= << >> +// /\ Len(log[i]) = m.mprevLogIndex +// /\ log' = [log EXCEPT ![i] = +// Append(log[i], m.mentries[1])] +// /\ UNCHANGED <> +// /\ UNCHANGED <> +// + + +} diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c new file mode 100644 index 0000000000..4a9055e172 --- /dev/null +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncAppendEntriesReply.h" +#include "sync.h" + +void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { + +// TLA+ Spec +//HandleAppendEntriesResponse(i, j, m) == +// /\ m.mterm = currentTerm[i] +// /\ \/ /\ m.msuccess \* successful +// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] +// /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] +// \/ /\ \lnot m.msuccess \* not successful +// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = +// Max({nextIndex[i][j] - 1, 1})] +// /\ UNCHANGED <> +// /\ Discard(m) +// /\ UNCHANGED <> + +} diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncElection.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c new file mode 100644 index 0000000000..fbb969eb1c --- /dev/null +++ b/source/libs/sync/src/syncMain.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include "sync.h" +#include "syncInt.h" + +int32_t syncInit() { return 0; } + +void syncCleanUp() {} + +int64_t syncStart(const SSyncInfo* pSyncInfo) { return 0; } + +void syncStop(int64_t rid) {} + +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; } + +int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { return 0; } + +ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } + +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c new file mode 100644 index 0000000000..dcfc940f76 --- /dev/null +++ b/source/libs/sync/src/syncMessage.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncMessage.h" +#include "sync.h" +#include "syncRaft.h" + +void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncOnMessage.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c new file mode 100644 index 0000000000..85c2c6fe27 --- /dev/null +++ b/source/libs/sync/src/syncRaft.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaft.h" +#include "sync.h" + +int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } + +static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncRaftEntry.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c new file mode 100644 index 0000000000..4a5fc201b0 --- /dev/null +++ b/source/libs/sync/src/syncRaftLog.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaftLog.h" +#include "sync.h" + +int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } + +// get one log entry, user need to free pBuf->data +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf) { return 0; } + +// update log store commit index with "index" +int32_t raftLogupdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; } + +// truncate log with index, entries after the given index (>index) will be deleted +int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; } + +// return commit index of log +SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore) { return 0; } + +// return index of last entry +SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore) { return 0; } + +// return term of last entry +SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c new file mode 100644 index 0000000000..d45e53132c --- /dev/null +++ b/source/libs/sync/src/syncRaftStore.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaftStore.h" +#include "sync.h" + +int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } + +int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } + +int32_t voteFor(SRaftId *pRaftId) { return 0; } + +int32_t persistVoteFor(SRaftId *pRaftId) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncReplication.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c new file mode 100644 index 0000000000..c31ec0f34d --- /dev/null +++ b/source/libs/sync/src/syncRequestVote.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRequestVote.h" +#include "sync.h" + +void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { + +// TLA+ Spec +//RequestVote(i, j) == +// /\ state[i] = Candidate +// /\ j \notin votesResponded[i] +// /\ Send([mtype |-> RequestVoteRequest, +// mterm |-> currentTerm[i], +// mlastLogTerm |-> LastTerm(log[i]), +// mlastLogIndex |-> Len(log[i]), +// msource |-> i, +// mdest |-> j]) +// /\ UNCHANGED <> + +} + +void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { + +// TLA+ Spec +//HandleRequestVoteRequest(i, j, m) == +// LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) +// \/ /\ m.mlastLogTerm = LastTerm(log[i]) +// /\ m.mlastLogIndex >= Len(log[i]) +// grant == /\ m.mterm = currentTerm[i] +// /\ logOk +// /\ votedFor[i] \in {Nil, j} +// IN /\ m.mterm <= currentTerm[i] +// /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] +// \/ ~grant /\ UNCHANGED votedFor +// /\ Reply([mtype |-> RequestVoteResponse, +// mterm |-> currentTerm[i], +// mvoteGranted |-> grant, +// \* mlog is used just for the `elections' history variable for +// \* the proof. It would not exist in a real implementation. +// mlog |-> log[i], +// msource |-> i, +// mdest |-> j], +// m) +// /\ UNCHANGED <> + +} diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c new file mode 100644 index 0000000000..ba9787f00c --- /dev/null +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRequestVoteReply.h" +#include "sync.h" + +void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { + +// TLA+ Spec +//HandleRequestVoteResponse(i, j, m) == +// \* This tallies votes even when the current state is not Candidate, but +// \* they won't be looked at, so it doesn't matter. +// /\ m.mterm = currentTerm[i] +// /\ votesResponded' = [votesResponded EXCEPT ![i] = +// votesResponded[i] \cup {j}] +// /\ \/ /\ m.mvoteGranted +// /\ votesGranted' = [votesGranted EXCEPT ![i] = +// votesGranted[i] \cup {j}] +// /\ voterLog' = [voterLog EXCEPT ![i] = +// voterLog[i] @@ (j :> m.mlog)] +// \/ /\ ~m.mvoteGranted +// /\ UNCHANGED <> +// /\ Discard(m) +// /\ UNCHANGED <> + +} diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c new file mode 100644 index 0000000000..8a27f097d1 --- /dev/null +++ b/source/libs/sync/src/syncSnapshot.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncSnapshot.h" +#include "sync.h" +#include "syncRaft.h" + +int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } + +int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c new file mode 100644 index 0000000000..206dd70046 --- /dev/null +++ b/source/libs/sync/src/syncTimeout.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncTimeout.h" +#include "sync.h" + +void onTimeout(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp new file mode 100644 index 0000000000..47566d537e --- /dev/null +++ b/source/libs/sync/test/syncTest.cpp @@ -0,0 +1,7 @@ +#include + +int main() { + printf("test \n"); + return 0; +} + diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index d5a8cf5f84..d4d9bff56c 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -217,7 +217,7 @@ typedef struct SConnBuffer { char* buf; int len; int cap; - int left; + int total; } SConnBuffer; typedef void (*AsyncCB)(uv_async_t* handle); @@ -238,10 +238,11 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) void transDestroyAsyncPool(SAsyncPool* pool); int transSendAsync(SAsyncPool* pool, queue* mq); -int transInitBuffer(SConnBuffer* buf); -int transClearBuffer(SConnBuffer* buf); -int transDestroyBuffer(SConnBuffer* buf); -int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +int transInitBuffer(SConnBuffer* buf); +int transClearBuffer(SConnBuffer* buf); +int transDestroyBuffer(SConnBuffer* buf); +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +bool transReadComplete(SConnBuffer* connBuf); // int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 3c8c922d83..a36b671eb4 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -56,6 +56,7 @@ typedef struct { int8_t connType; int64_t index; char label[TSDB_LABEL_LEN]; + bool noPool; // pool or not char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 3bb7d103d7..72c1ff6893 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -64,6 +64,7 @@ typedef struct { void (*cfp)(void *parent, SRpcMsg *, SEpSet *); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); + bool noPool; int32_t refCount; void * parent; void * idPool; // handle to ID pool diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c3d3cfa2ab..48c15ca286 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -27,7 +27,7 @@ void* rpcOpen(const SRpcInit* pInit) { return NULL; } if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } pRpc->cfp = pInit->cfp; if (pInit->connType == TAOS_CONN_SERVER) { @@ -35,6 +35,8 @@ void* rpcOpen(const SRpcInit* pInit) { } else { pRpc->numOfThreads = pInit->numOfThreads; } + + pRpc->noPool = pInit->noPool; pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f1bd1ba980..8312c0217c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -84,8 +84,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// check whether already read complete packet from server -static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after read nbytes from socket @@ -126,6 +124,9 @@ static void clientHandleResp(SCliConn* conn) { pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); + // buf's mem alread translated to rpcMsg.pCont + transClearBuffer(&conn->readBuf); + SRpcMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = transContFromHead((char*)pHead); @@ -134,15 +135,15 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.ahandle = pCtx->ahandle; if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY) { + rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); } - tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), - inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), - ntohs(conn->locaddr.sin_port)); + tDebug("%s client conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pRpc->label, conn, + TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), rpcMsg.contLen); conn->secured = pHead->secured; if (conn->push != NULL && conn->ctnRdCnt != 0) { @@ -150,26 +151,26 @@ static void clientHandleResp(SCliConn* conn) { conn->push = NULL; } else { if (pCtx->pSem == NULL) { - tTrace("client conn %p handle resp", conn); + tTrace("%s client conn %p handle resp", pRpc->label, conn); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); } else { - tTrace("client conn(sync) %p handle resp", conn); + tTrace("%s client conn(sync) %p handle resp", pRpc->label, conn); memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg)); tsem_post(pCtx->pSem); } } conn->ctnRdCnt += 1; - // buf's mem alread translated to rpcMsg.pCont - transClearBuffer(&conn->readBuf); - uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb); SCliThrdObj* pThrd = conn->hostThrd; // user owns conn->persist = 1 if (conn->push == NULL && conn->persist == 0) { - addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + if (pRpc->noPool == true) { + } else { + addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + } } destroyCmsg(conn->data); conn->data = NULL; @@ -184,7 +185,6 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - tTrace("client conn %p start to destroy", pConn); SCliMsg* pMsg = pConn->data; tmsg_t msgType = TDMT_MND_CONNECT; @@ -213,6 +213,7 @@ static void clientHandleExcept(SCliConn* pConn) { } pConn->push = NULL; } + tTrace("%s client conn %p start to destroy", pCtx->pTransInst->label, pConn); if (pConn->push == NULL) { destroyCmsg(pConn->data); pConn->data = NULL; @@ -226,7 +227,7 @@ static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; - tTrace("client conn timeout, try to remove expire conn from conn pool"); + tTrace("%s, client conn timeout, try to remove expire conn from conn pool", pRpc->label); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); while (p != NULL) { @@ -306,23 +307,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); } -static bool clientReadComplete(SConnBuffer* data) { - STransMsgHead head; - int32_t headLen = sizeof(head); - if (data->len >= headLen) { - memcpy((char*)&head, data->buf, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); - if (msgLen > data->len) { - data->left = msgLen - data->len; - return false; - } else if (msgLen == data->len) { - data->left = 0; - return true; - } - } else { - return false; - } -} static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; @@ -337,8 +321,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { pBuf->len += nread; - if (clientReadComplete(pBuf)) { - uv_read_stop((uv_stream_t*)conn->stream); + if (transReadComplete(pBuf)) { tTrace("client conn %p read complete", conn); clientHandleResp(conn); } else { @@ -346,6 +329,10 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf } return; } + if (nread == UV_EOF) { + tError("client conn %p read error: %s", conn, uv_err_name(nread)); + clientHandleExcept(conn); + } assert(nread <= 0); if (nread == 0) { // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb @@ -353,7 +340,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf // read(2). return; } - if (nread < 0 || nread == UV_EOF) { + if (nread < 0) { tError("client conn %p read error: %s", conn, uv_err_name(nread)); clientHandleExcept(conn); } @@ -467,6 +454,7 @@ static void clientConnCb(uv_connect_t* req, int status) { static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("client work thread %p start to quit", pThrd); destroyCmsg(pMsg); + destroyConnPool(pThrd->pool); // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); uv_timer_stop(pThrd->timer); pThrd->quit = true; @@ -483,7 +471,10 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = NULL; if (pMsg->msg.handle == NULL) { - conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + if (pCtx->pTransInst->noPool == true) { + } else { + conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + } if (conn != NULL) { tTrace("client conn %p get from conn pool", conn); } @@ -512,7 +503,11 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; - + uv_tcp_nodelay((uv_tcp_t*)conn->stream, 1); + int ret = uv_tcp_keepalive((uv_tcp_t*)conn->stream, 1, 1); + if (ret) { + tTrace("client conn %p failed to set keepalive, %s", conn, uv_err_name(ret)); + } // write req handle conn->writeReq = malloc(sizeof(uv_write_t)); conn->writeReq->data = conn; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7aa5aa16f1..9a8607b0ed 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -205,6 +205,7 @@ int transInitBuffer(SConnBuffer* buf) { } int transClearBuffer(SConnBuffer* buf) { memset(buf, 0, sizeof(*buf)); + buf->total = -1; return 0; } int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { @@ -214,32 +215,38 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user * info--->| */ - static const int CAPACITY = 1024; + static const int CAPACITY = sizeof(STransMsgHead); SConnBuffer* p = connBuf; if (p->cap == 0) { p->buf = (char*)calloc(CAPACITY, sizeof(char)); p->len = 0; p->cap = CAPACITY; - p->left = -1; + p->total = -1; uvBuf->base = p->buf; uvBuf->len = CAPACITY; } else { - if (p->len >= p->cap) { - if (p->left == -1) { - p->cap *= 2; - p->buf = realloc(p->buf, p->cap); - } else if (p->len + p->left > p->cap) { - p->cap = p->len + p->left; - p->buf = realloc(p->buf, p->len + p->left); - } - } + p->cap = p->total; + p->buf = realloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; } return 0; } +// check whether already read complete +bool transReadComplete(SConnBuffer* connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy((char*)&head, connBuf->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + connBuf->total = msgLen; + } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} int transUnpackMsg(STransMsgHead* msgHead) {} diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 7ddeb99c9d..9fca371bf3 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -61,6 +61,7 @@ typedef struct SWorkThrdObj { SAsyncPool* asyncPool; // uv_async_t* workerAsync; // queue msg; + queue conn; pthread_mutex_t msgMtx; void* pTransInst; } SWorkThrdObj; @@ -95,6 +96,7 @@ static void uvOnAcceptCb(uv_stream_t* stream, int status); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); +static void uvShutDownCb(uv_shutdown_t* req, int status); static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); @@ -102,8 +104,7 @@ static void uvStartSendResp(SSrvMsg* msg); static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet -static bool readComplete(SConnBuffer* buf); -static SSrvConn* createConn(); +static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); static void uvDestroyConn(uv_handle_t* handle); @@ -117,51 +118,11 @@ static bool addHandleToWorkloop(void* arg); static bool addHandleToAcceptloop(void* arg); void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - /* - * formate of data buffer: - * |<--------------------------data from socket------------------------------->| - * |<------STransMsgHead------->|<-------------------other data--------------->| - */ SSrvConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; transAllocBuffer(pBuf, buf); } -// check data read from socket complete or not -// -static bool readComplete(SConnBuffer* data) { - // TODO(yihao): handle pipeline later - STransMsgHead head; - int32_t headLen = sizeof(head); - if (data->len >= headLen) { - memcpy((char*)&head, data->buf, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); - if (msgLen > data->len) { - data->left = msgLen - data->len; - return false; - } else if (msgLen == data->len) { - return true; - } else if (msgLen < data->len) { - return false; - // handle other packet later - } - } else { - return false; - } -} - -// static void uvDoProcess(SRecvInfo* pRecv) { -// // impl later -// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; -// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; -// SSrvConn* pConn = pRecv->thandle; -// tDump(pRecv->msg, pRecv->msgLen); -// terrno = 0; -// // SRpcReqContext* pContest; -// -// // do auth and check -//} - static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; @@ -241,7 +202,7 @@ static void uvHandleReq(SSrvConn* pConn) { } pConn->inType = pHead->msgType; - assert(transIsReq(pHead->msgType)); + // assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; pHead->code = htonl(pHead->code); @@ -266,9 +227,9 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->ref++; - tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), - ntohs(pConn->locaddr.sin_port)); + ntohs(pConn->locaddr.sin_port), rpcMsg.contLen); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -282,7 +243,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; tTrace("server conn %p read summary, total read: %d, current read: %d", conn, pBuf->len, (int)nread); - if (readComplete(pBuf)) { + if (transReadComplete(pBuf)) { tTrace("server conn %p alread read complete packet", conn); uvHandleReq(conn); } else { @@ -290,6 +251,14 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } + if (nread == UV_EOF) { + tError("server conn %p read error: %s", conn, uv_err_name(nread)); + if (conn->ref > 1) { + conn->ref++; // ref > 1 signed that write is in progress + } + destroyConn(conn, true); + return; + } if (nread == 0) { return; } @@ -302,8 +271,8 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - buf->base = malloc(sizeof(char)); buf->len = 2; + buf->base = calloc(1, sizeof(char) * buf->len); } void uvOnTimeoutCb(uv_timer_t* handle) { @@ -386,6 +355,7 @@ static void uvStartSendRespInternal(SSrvMsg* smsg) { static void uvStartSendResp(SSrvMsg* smsg) { // impl SSrvConn* pConn = smsg->pConn; + pConn->ref--; // if (taosArrayGetSize(pConn->srvMsgs) > 0) { tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); @@ -403,6 +373,16 @@ static void destroySmsg(SSrvMsg* smsg) { transFreeMsg(smsg->msg.pCont); free(smsg); } +static void destroyAllConn(SWorkThrdObj* pThrd) { + while (!QUEUE_IS_EMPTY(&pThrd->conn)) { + queue* h = QUEUE_HEAD(&pThrd->conn); + QUEUE_REMOVE(h); + QUEUE_INIT(h); + + SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue); + destroyConn(c, true); + } +} void uvWorkerAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; SWorkThrdObj* pThrd = item->pThrd; @@ -424,8 +404,11 @@ void uvWorkerAsyncCb(uv_async_t* handle) { continue; } if (msg->pConn == NULL) { - // free(msg); + + destroyAllConn(pThrd); + + uv_loop_close(pThrd->loop); uv_stop(pThrd->loop); } else { uvStartSendResp(msg); @@ -439,9 +422,16 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } static void uvAcceptAsyncCb(uv_async_t* async) { SServerObj* srv = async->data; + uv_close((uv_handle_t*)&srv->server, NULL); uv_stop(srv->loop); } +static void uvShutDownCb(uv_shutdown_t* req, int status) { + tDebug("conn failed to shut down: %s", uv_err_name(status)); + uv_close((uv_handle_t*)req->handle, uvDestroyConn); + free(req); +} + void uvOnAcceptCb(uv_stream_t* stream, int status) { if (status == -1) { return; @@ -491,7 +481,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SSrvConn* pConn = createConn(); + SSrvConn* pConn = createConn(pThrd); pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ @@ -507,6 +497,9 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; + // uv_tcp_nodelay(pConn->pTcp, 1); + // uv_tcp_keepalive(pConn->pTcp, 1, 1); + // init write request, just pConn->pWriter = calloc(1, sizeof(uv_write_t)); pConn->pWriter->data = pConn; @@ -560,6 +553,9 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); + // conn set + QUEUE_INIT(&pThrd->conn); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; @@ -598,8 +594,13 @@ void* workerThread(void* arg) { uv_run(pThrd->loop, UV_RUN_DEFAULT); } -static SSrvConn* createConn() { +static SSrvConn* createConn(void* hThrd) { + SWorkThrdObj* pThrd = hThrd; + SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + QUEUE_INIT(&pConn->queue); + + QUEUE_PUSH(&pThrd->conn, &pConn->queue); pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // tTrace("conn %p created", pConn); ++pConn->ref; @@ -610,7 +611,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { if (conn == NULL) { return; } - tTrace("server conn %p try to destroy", conn); + tTrace("server conn %p try to destroy, ref: %d", conn, conn->ref); if (--conn->ref > 0) { return; } @@ -621,20 +622,23 @@ static void destroyConn(SSrvConn* conn, bool clear) { destroySmsg(msg); } taosArrayDestroy(conn->srvMsgs); - - // destroySmsg(conn->pSrvMsg); - // conn->pSrvMsg = NULL; + QUEUE_REMOVE(&conn->queue); if (clear) { - uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); + tTrace("try to destroy conn %p", conn); + uv_tcp_close_reset(conn->pTcp, uvDestroyConn); + // uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); + // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); + // uv_unref((uv_handle_t*)conn->pTcp); + // uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); } } static void uvDestroyConn(uv_handle_t* handle) { SSrvConn* conn = handle->data; tDebug("server conn %p destroy", conn); uv_timer_stop(conn->pTimer); - free(conn->pTimer); - // free(conn->pTcp); + // free(conn->pTimer); + free(conn->pTcp); free(conn->pWriter); free(conn); } diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 6f80ea42ac..d1fefe2c72 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -16,69 +16,168 @@ #include #include #include "tep.h" +#include "tglobal.h" #include "trpc.h" +#include "ulog.h" using namespace std; -class TransObj { - public: - TransObj() { - const char *label = "APP"; - const char *secret = "secret"; - const char *user = "user"; - const char *ckey = "ckey"; +const char *label = "APP"; +const char *secret = "secret"; +const char *user = "user"; +const char *ckey = "ckey"; +class Server; +int port = 7000; +// server process +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +class Client { + public: + void Init(int nThread) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = (char *)label; - rpcInit.numOfThreads = 5; - rpcInit.cfp = NULL; - rpcInit.sessions = 100; - rpcInit.idleTime = 100; + rpcInit.numOfThreads = nThread; + rpcInit.cfp = processResp; rpcInit.user = (char *)user; rpcInit.secret = (char *)secret; rpcInit.ckey = (char *)ckey; rpcInit.spi = 1; - } - bool startCli() { - trans = NULL; + rpcInit.parent = this; rpcInit.connType = TAOS_CONN_CLIENT; - trans = rpcOpen(&rpcInit); - return trans != NULL ? true : false; + this->transCli = rpcOpen(&rpcInit); + tsem_init(&this->sem, 0, 0); } - bool startSrv() { - trans = NULL; - rpcInit.connType = TAOS_CONN_SERVER; - trans = rpcOpen(&rpcInit); - return trans != NULL ? true : false; + void SetResp(SRpcMsg *pMsg) { + // set up resp; + this->resp = *pMsg; + } + SRpcMsg *Resp() { return &this->resp; } + + void Restart() { + rpcClose(this->transCli); + this->transCli = rpcOpen(&rpcInit); } - bool sendAndRecv() { + void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { SEpSet epSet = {0}; epSet.inUse = 0; - addEpIntoEpSet(&epSet, "192.168.1.1", 7000); - addEpIntoEpSet(&epSet, "192.168.0.1", 7000); + addEpIntoEpSet(&epSet, "127.0.0.1", 7000); - if (trans == NULL) { - return false; - } - SRpcMsg rpcMsg = {0}, reqMsg = {0}; - reqMsg.pCont = rpcMallocCont(10); - reqMsg.contLen = 10; - reqMsg.ahandle = NULL; - rpcSendRecv(trans, &epSet, &reqMsg, &rpcMsg); - int code = rpcMsg.code; - std::cout << tstrerror(code) << std::endl; - return true; + rpcSendRequest(this->transCli, &epSet, req, NULL); + SemWait(); + *resp = this->resp; } - bool stop() { - rpcClose(trans); - trans = NULL; - return true; + void SemWait() { tsem_wait(&this->sem); } + void SemPost() { tsem_post(&this->sem); } + void Reset() {} + + ~Client() { + if (this->transCli) rpcClose(this->transCli); } private: - void * trans; + tsem_t sem; SRpcInit rpcInit; + void * transCli; + SRpcMsg resp; +}; +class Server { + public: + Server() { + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = port; + rpcInit.label = (char *)label; + rpcInit.numOfThreads = 5; + rpcInit.cfp = processReq; + rpcInit.user = (char *)user; + rpcInit.secret = (char *)secret; + rpcInit.ckey = (char *)ckey; + rpcInit.spi = 1; + rpcInit.connType = TAOS_CONN_SERVER; + } + void Start() { + this->transSrv = rpcOpen(&this->rpcInit); + taosMsleep(1000); + } + void Stop() { + if (this->transSrv == NULL) return; + rpcClose(this->transSrv); + this->transSrv = NULL; + } + void Restart() { + this->Stop(); + this->Start(); + } + ~Server() { + if (this->transSrv) rpcClose(this->transSrv); + this->transSrv = NULL; + } + + private: + SRpcInit rpcInit; + void * transSrv; +}; +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + Client *client = (Client *)parent; + client->SetResp(pMsg); + client->SemPost(); +} +class TransObj { + public: + TransObj() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + cqDebugFlag = 0; + tscEmbeddedInUtil = 1; + tsAsyncLog = 0; + + std::string path = "/tmp/transport"; + taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + } + cli = new Client; + cli->Init(1); + srv = new Server; + srv->Start(); + } + void RestartCli() { cli->Restart(); } + void StopSrv() { srv->Stop(); } + void RestartSrv() { srv->Restart(); } + void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); } + ~TransObj() { + delete cli; + delete srv; + } + + private: + Client *cli; + Server *srv; }; class TransEnv : public ::testing::Test { protected: @@ -93,11 +192,34 @@ class TransEnv : public ::testing::Test { TransObj *tr = NULL; }; -TEST_F(TransEnv, test_start_stop) { - assert(tr->startCli()); - assert(tr->sendAndRecv()); - assert(tr->stop()); - assert(tr->startSrv()); - assert(tr->stop()); +// TEST_F(TransEnv, 01sendAndRec) { +// for (int i = 0; i < 1; i++) { +// SRpcMsg req = {0}, resp = {0}; +// req.msgType = 0; +// req.pCont = rpcMallocCont(10); +// req.contLen = 10; +// tr->cliSendAndRecv(&req, &resp); +// assert(resp.code == 0); +// } +//} + +TEST_F(TransEnv, 02StopServer) { + for (int i = 0; i < 1; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } + SRpcMsg req = {0}, resp = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->StopSrv(); + // tr->RestartSrv(); + tr->cliSendAndRecv(&req, &resp); + + assert(resp.code != 0); } diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 7e542ef80f..bb526e0ba0 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -22,13 +22,12 @@ * windows implementation */ - -#include #include -#include +#include #include +#include -#pragma warning( disable : 4244 ) +#pragma warning(disable : 4244) typedef void (*win_timer_f)(int signo); @@ -40,8 +39,8 @@ void WINAPI taosWinOnTimer(UINT wTimerID, UINT msg, DWORD_PTR dwUser, DWORD_PTR } static MMRESULT timerId; -int taosInitTimer(win_timer_f callback, int ms) { - DWORD_PTR param = *((int64_t *) & callback); +int taosInitTimer(win_timer_f callback, int ms) { + DWORD_PTR param = *((int64_t *)&callback); timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); if (timerId == 0) { @@ -50,9 +49,7 @@ int taosInitTimer(win_timer_f callback, int ms) { return 0; } -void taosUninitTimer() { - timeKillEvent(timerId); -} +void taosUninitTimer() { timeKillEvent(timerId); } #elif defined(_TD_DARWIN_64) @@ -60,32 +57,32 @@ void taosUninitTimer() { * darwin implementation */ -#include #include +#include #include static void (*timer_callback)(int); -static int timer_ms = 0; -static pthread_t timer_thread; -static int timer_kq = -1; -static volatile int timer_stop = 0; +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; -static void* timer_routine(void *arg) { +static void* timer_routine(void* arg) { (void)arg; setThreadName("timer"); - int r = 0; + int r = 0; struct timespec to = {0}; - to.tv_sec = timer_ms / 1000; - to.tv_nsec = (timer_ms % 1000) * 1000000; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; while (!timer_stop) { struct kevent64_s kev[10] = {0}; - r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); - if (r!=0) { + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev) / sizeof(kev[0]), 0, &to); + if (r != 0) { fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); abort(); } - timer_callback(SIGALRM); // just mock + timer_callback(SIGALRM); // just mock } return NULL; @@ -93,11 +90,13 @@ static void* timer_routine(void *arg) { int taosInitTimer(void (*callback)(int), int ms) { int r = 0; - timer_ms = ms; + timer_kq = -1; + timer_stop = 0; + timer_ms = ms; timer_callback = callback; timer_kq = kqueue(); - if (timer_kq==-1) { + if (timer_kq == -1) { fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); @@ -144,10 +143,10 @@ static void taosDeleteTimer(void *tharg) { timer_delete(*pTimer); } -static pthread_t timerThread; -static timer_t timerId; +static pthread_t timerThread; +static timer_t timerId; static volatile bool stopTimer = false; -static void *taosProcessAlarmSignal(void *tharg) { +static void * taosProcessAlarmSignal(void *tharg) { // Block the signal sigset_t sigset; sigemptyset(&sigset); @@ -159,18 +158,18 @@ static void *taosProcessAlarmSignal(void *tharg) { setThreadName("tmr"); - #ifdef _ALPINE - sevent.sigev_notify = SIGEV_THREAD; - sevent.sigev_value.sival_int = syscall(__NR_gettid); - #else - sevent.sigev_notify = SIGEV_THREAD_ID; - sevent._sigev_un._tid = syscall(__NR_gettid); - #endif - +#ifdef _ALPINE + sevent.sigev_notify = SIGEV_THREAD; + sevent.sigev_value.sival_int = syscall(__NR_gettid); +#else + sevent.sigev_notify = SIGEV_THREAD_ID; + sevent._sigev_un._tid = syscall(__NR_gettid); +#endif + sevent.sigev_signo = SIGALRM; if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) { - //printf("Failed to create timer"); + // printf("Failed to create timer"); } pthread_cleanup_push(taosDeleteTimer, &timerId); @@ -182,36 +181,37 @@ static void *taosProcessAlarmSignal(void *tharg) { ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; if (timer_settime(timerId, 0, &ts, NULL)) { - //printf("Failed to init timer"); + // printf("Failed to init timer"); return NULL; } int signo; while (!stopTimer) { if (sigwait(&sigset, &signo)) { - //printf("Failed to wait signal: number %d", signo); + // printf("Failed to wait signal: number %d", signo); continue; } /* //printf("Signal handling: number %d ......\n", signo); */ callback(0); } - + pthread_cleanup_pop(1); return NULL; } int taosInitTimer(void (*callback)(int), int ms) { + stopTimer = false; pthread_attr_t tattr; pthread_attr_init(&tattr); int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); pthread_attr_destroy(&tattr); if (code != 0) { - //printf("failed to create timer thread"); + // printf("failed to create timer thread"); return -1; } else { - //printf("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); + // printf("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); } return 0; @@ -220,7 +220,7 @@ int taosInitTimer(void (*callback)(int), int ms) { void taosUninitTimer() { stopTimer = true; - //printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); + // printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); pthread_join(timerThread, NULL); } diff --git a/source/util/src/compare.c b/source/util/src/compare.c index a1c1625c34..6124de9459 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -23,23 +23,51 @@ #include "thash.h" #include "types.h" #include "ulog.h" +#include "tdef.h" +#include "taos.h" -int32_t setCompareBytes1(const void *pLeft, const void *pRight) { +int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; } -int32_t setCompareBytes2(const void *pLeft, const void *pRight) { +int32_t setChkInBytes2(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; } -int32_t setCompareBytes4(const void *pLeft, const void *pRight) { +int32_t setChkInBytes4(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; } -int32_t setCompareBytes8(const void *pLeft, const void *pRight) { +int32_t setChkInBytes8(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; } +int32_t setChkNotInBytes1(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; +} + +int32_t setChkNotInBytes2(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; +} + +int32_t setChkNotInBytes4(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; +} + +int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; +} + + +int32_t compareChkInString(const void *pLeft, const void* pRight) { + return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} + +int32_t compareChkNotInString(const void *pLeft, const void* pRight) { + return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} + + int32_t compareInt8Val(const void *pLeft, const void *pRight) { int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); if (left > right) return 1; @@ -393,6 +421,156 @@ int32_t taosArrayCompareString(const void* a, const void* b) { return compareLenPrefixedStr(x, y); } -int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { - return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; + +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); + char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); + free(pattern); + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } + +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); + + wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + + int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); + free(pattern); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + + +__compar_fn_t getComparFunc(int32_t type, int32_t optr) { + __compar_fn_t comparFn = NULL; + + if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkInBytes8; + default: + assert(0); + } + } + + if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkNotInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkNotInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkNotInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkNotInBytes8; + default: + assert(0); + } + } + + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; + case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_BINARY: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternMatch; + } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternNotMatch; + } else if (optr == TSDB_RELATION_IN) { + comparFn = compareChkInString; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = compareChkNotInString; + } else { /* normal relational comparFn */ + comparFn = compareLenPrefixedStr; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == TSDB_RELATION_LIKE) { + comparFn = compareWStrPatternMatch; + } else if (optr == TSDB_RELATION_NOT_LIKE) { + comparFn = compareWStrPatternNotMatch; + } else if (optr == TSDB_RELATION_IN) { + comparFn = compareChkInString; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = compareChkNotInString; + } else { + comparFn = compareLenPrefixedWStr; + } + break; + } + + case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; + case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} + + diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index e91b11026b..2d7f8a5b03 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -164,20 +164,20 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { return fn; } -//int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { -// return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); -//} -// -//int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { -// return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); -//} +int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); +} + +int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); +} _equal_fn_t taosGetDefaultEqualFunction(int32_t type) { _equal_fn_t fn = NULL; -// switch (type) { -// case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; -// case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; -// default: fn = memcmp; break; -// } + switch (type) { + case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; + case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; + default: fn = memcmp; break; + } return fn; } diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 556e8f8060..13367843fc 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -32,6 +32,10 @@ int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t return tjsonAddStringToObject(pJson, pName, tmp); } +int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) { + return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); +} + int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } @@ -74,3 +78,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } + +char* tjsonToUnformattedString(const SJson* pJson) { + return cJSON_PrintUnformatted((cJSON*)pJson); +} \ No newline at end of file diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 1fdc2257d7..65101a5e07 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -13,19 +13,49 @@ * along with this program. If not, see . */ +#include "ttimer.h" #include "os.h" +#include "taoserror.h" #include "tlog.h" #include "tsched.h" -#include "ttimer.h" #include "tutil.h" -#include "taoserror.h" -#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} -#define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrFatal(...) \ + { \ + if (tmrDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrError(...) \ + { \ + if (tmrDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrWarn(...) \ + { \ + if (tmrDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrInfo(...) \ + { \ + if (tmrDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrDebug(...) \ + { \ + if (tmrDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } +#define tmrTrace(...) \ + { \ + if (tmrDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ + } \ + } #define TIMER_STATE_WAITING 0 #define TIMER_STATE_EXPIRED 1 @@ -81,7 +111,7 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tmrDebugFlag = 131; +int32_t tmrDebugFlag = 131; uint32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; @@ -91,7 +121,7 @@ static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; static int numOfTmrCtrl = 0; -int taosTmrThreads = 1; +int taosTmrThreads = 1; static uintptr_t nextTimerId = 0; static time_wheel_t wheels[] = { @@ -119,7 +149,7 @@ static void timerDecRef(tmr_obj_t* timer) { static void lockTimerList(timer_list_t* list) { int64_t tid = taosGetSelfPthreadId(); - int i = 0; + int i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { sched_yield(); @@ -276,11 +306,11 @@ static void addToExpired(tmr_obj_t* head) { const char* fmt = "%s adding expired timer[id=%" PRIuPTR ", fp=%p, param=%p] to queue."; while (head != NULL) { - uintptr_t id = head->id; + uintptr_t id = head->id; tmr_obj_t* next = head->next; tmrDebug(fmt, head->ctrl->label, id, head->fp, head->param); - SSchedMsg schedMsg; + SSchedMsg schedMsg; schedMsg.fp = NULL; schedMsg.tfp = processExpiredTimer; schedMsg.msg = NULL; @@ -491,6 +521,8 @@ static void taosTmrModuleInit(void) { return; } + memset(&timerMap, 0, sizeof(timerMap)); + for (uint32_t i = 0; i < tsMaxTmrCtrl - 1; ++i) { tmr_ctrl_t* ctrl = tmrCtrls + i; ctrl->next = ctrl + 1; @@ -570,7 +602,8 @@ void taosTmrCleanUp(void* handle) { unusedTmrCtrl = ctrl; pthread_mutex_unlock(&tmrCtrlMutex); - if (numOfTmrCtrl <=0) { + tmrDebug("time controller's tmr ctrl size: %d", numOfTmrCtrl); + if (numOfTmrCtrl <= 0) { taosUninitTimer(); taosCleanUpScheduler(tmrQhandle); @@ -585,7 +618,7 @@ void taosTmrCleanUp(void* handle) { for (size_t i = 0; i < timerMap.size; i++) { timer_list_t* list = timerMap.slots + i; - tmr_obj_t* t = list->timers; + tmr_obj_t* t = list->timers; while (t != NULL) { tmr_obj_t* next = t->mnext; free(t); @@ -595,6 +628,8 @@ void taosTmrCleanUp(void* handle) { free(timerMap.slots); free(tmrCtrls); - tmrDebug("timer module is cleaned up"); + tmrCtrls = NULL; + unusedTmrCtrl = NULL; + tmrModuleInit = PTHREAD_ONCE_INIT; // to support restart } }