xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/tq

This commit is contained in:
Liu Jicong 2022-02-23 16:13:53 +08:00
parent 302463f635 f6edfd755b
commit ae6cb518e6
47 changed files with 4000 additions and 865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/common/ttypes.h
View File

@ -91,6 +91,8 @@ typedef struct {
do { \
switch (_type) { \
case TSDB_DATA_TYPE_BOOL: \
*(bool *)(_v) = (bool)(_data); \
break; \
case TSDB_DATA_TYPE_TINYINT: \
*(int8_t *)(_v) = (int8_t)(_data); \
break; \
@ -104,6 +106,7 @@ typedef struct {
*(uint16_t *)(_v) = (uint16_t)(_data); \
break; \
case TSDB_DATA_TYPE_BIGINT: \
case TSDB_DATA_TYPE_TIMESTAMP: \
*(int64_t *)(_v) = (int64_t)(_data); \
break; \
case TSDB_DATA_TYPE_UBIGINT: \

1
include/libs/function/function.h
View File

@ -228,6 +228,7 @@ typedef struct SAggFunctionInfo {
typedef struct SScalarParam {
void* data;
bool colData;
int32_t num;
int32_t type;
int32_t bytes;

15
include/libs/nodes/nodes.h
View File

@ -61,22 +61,27 @@ typedef enum ENodeType {
QUERY_NODE_INTERVAL_WINDOW,
QUERY_NODE_NODE_LIST,
QUERY_NODE_FILL,
QUERY_NODE_RAW_EXPR, // Only be used in parser module.
QUERY_NODE_COLUMN_REF,
QUERY_NODE_TARGET,
// Only be used in parser module.
QUERY_NODE_RAW_EXPR,
QUERY_NODE_TUPLE_DESC,
QUERY_NODE_SLOT_DESC,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR,
QUERY_NODE_SELECT_STMT,
QUERY_NODE_SHOW_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN,
QUERY_NODE_LOGIC_PLAN_JOIN,
QUERY_NODE_LOGIC_PLAN_FILTER,
QUERY_NODE_LOGIC_PLAN_AGG,
QUERY_NODE_LOGIC_PLAN_PROJECT
QUERY_NODE_LOGIC_PLAN_PROJECT,
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT
} ENodeType;
/**

65
include/libs/nodes/plannodes.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif
#include "querynodes.h"
#include "tmsg.h"
typedef struct SLogicNode {
ENodeType type;
@ -31,10 +32,20 @@ typedef struct SLogicNode {
struct SLogicNode* pParent;
} SLogicNode;
typedef enum EScanType {
SCAN_TYPE_TAG,
SCAN_TYPE_TABLE,
SCAN_TYPE_STABLE,
SCAN_TYPE_STREAM
} EScanType;
typedef struct SScanLogicNode {
SLogicNode node;
SNodeList* pScanCols;
struct STableMeta* pMeta;
EScanType scanType;
uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow scanRange;
} SScanLogicNode;
typedef struct SJoinLogicNode {
@ -43,10 +54,6 @@ typedef struct SJoinLogicNode {
SNode* pOnConditions;
} SJoinLogicNode;
typedef struct SFilterLogicNode {
SLogicNode node;
} SFilterLogicNode;
typedef struct SAggLogicNode {
SLogicNode node;
SNodeList* pGroupKeys;
@ -58,6 +65,56 @@ typedef struct SProjectLogicNode {
SNodeList* pProjections;
} SProjectLogicNode;
typedef struct SSlotDescNode {
ENodeType type;
int16_t slotId;
SDataType dataType;
int16_t srcTupleId;
int16_t srcSlotId;
bool reserve;
bool output;
} SSlotDescNode;
typedef struct STupleDescNode {
ENodeType type;
int16_t tupleId;
SNodeList* pSlots;
} STupleDescNode;
typedef struct SPhysiNode {
ENodeType type;
STupleDescNode outputTuple;
SNode* pConditions;
SNodeList* pChildren;
struct SPhysiNode* pParent;
} SPhysiNode;
typedef struct SScanPhysiNode {
SPhysiNode node;
SNodeList* pScanCols;
uint64_t uid; // unique id of the table
int8_t tableType;
int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
int32_t count; // repeat count
int32_t reverse; // reverse scan count
} SScanPhysiNode;
typedef SScanPhysiNode SSystemTableScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode;
typedef struct STableScanPhysiNode {
SScanPhysiNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow scanRange;
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
typedef struct SProjectPhysiNode {
SPhysiNode node;
SNodeList* pProjections;
} SProjectPhysiNode;
#ifdef __cplusplus
}
#endif

55
include/libs/nodes/querynodes.h
View File

@ -68,6 +68,13 @@ typedef struct SColumnRefNode {
int16_t columnId;
} SColumnRefNode;
typedef struct STargetNode {
ENodeType type;
int16_t tupleId;
int16_t slotId;
SNode* pExpr;
} STargetNode;
typedef struct SValueNode {
SExprNode node; // QUERY_NODE_VALUE
char* literal;
@ -81,45 +88,6 @@ typedef struct SValueNode {
} datum;
} SValueNode;
typedef enum EOperatorType {
// arithmetic operator
OP_TYPE_ADD = 1,
OP_TYPE_SUB,
OP_TYPE_MULTI,
OP_TYPE_DIV,
OP_TYPE_MOD,
// bit operator
OP_TYPE_BIT_AND,
OP_TYPE_BIT_OR,
// comparison operator
OP_TYPE_GREATER_THAN,
OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL,
OP_TYPE_EQUAL,
OP_TYPE_NOT_EQUAL,
OP_TYPE_IN,
OP_TYPE_NOT_IN,
OP_TYPE_LIKE,
OP_TYPE_NOT_LIKE,
OP_TYPE_MATCH,
OP_TYPE_NMATCH,
OP_TYPE_IS_NULL,
OP_TYPE_IS_NOT_NULL,
OP_TYPE_IS_TRUE,
OP_TYPE_IS_FALSE,
OP_TYPE_IS_UNKNOWN,
OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN,
// json operator
OP_TYPE_JSON_GET_VALUE,
OP_TYPE_JSON_CONTAINS
} EOperatorType;
typedef struct SOperatorNode {
SExprNode node; // QUERY_NODE_OPERATOR
EOperatorType opType;
@ -127,11 +95,6 @@ typedef struct SOperatorNode {
SNode* pRight;
} SOperatorNode;
typedef enum ELogicConditionType {
LOGIC_COND_TYPE_AND,
LOGIC_COND_TYPE_OR,
LOGIC_COND_TYPE_NOT,
} ELogicConditionType;
typedef struct SLogicConditionNode {
SExprNode node; // QUERY_NODE_LOGIC_CONDITION
@ -141,6 +104,7 @@ typedef struct SLogicConditionNode {
typedef struct SNodeListNode {
ENodeType type; // QUERY_NODE_NODE_LIST
SDataType dataType;
SNodeList* pNodeList;
} SNodeListNode;
@ -306,7 +270,8 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
bool nodesIsTimeorderQuery(const SNode* pQuery);
bool nodesIsTimelineQuery(const SNode* pQuery);
void *nodesGetValueFromNode(SValueNode *pNode);
void* nodesGetValueFromNode(SValueNode *pNode);
#ifdef __cplusplus
}

18
include/libs/scalar/filter.h
View File

@ -20,6 +20,14 @@ extern "C" {
#endif
typedef struct SFilterInfo SFilterInfo;
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
enum {
FLT_OPTION_NO_REWRITE = 1,
FLT_OPTION_TIMESTAMP = 2,
FLT_OPTION_NEED_UNIQE = 4,
};
typedef struct SFilterColumnParam{
int32_t numOfCols;
@ -27,6 +35,16 @@ typedef struct SFilterColumnParam{
} SFilterColumnParam;
extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
#ifdef __cplusplus
}
#endif

1
include/libs/scalar/scalar.h
View File

@ -29,6 +29,7 @@ typedef struct SFilterInfo SFilterInfo;
int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
int32_t scalarGetOperatorParamNum(EOperatorType type);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut);

70
include/util/tdef.h
View File

@ -109,35 +109,52 @@ do { \
(src) = (void *)((char *)src + sizeof(type));\
} while(0)
typedef enum EOperatorType {
// arithmetic operator
OP_TYPE_ADD = 1,
OP_TYPE_SUB,
OP_TYPE_MULTI,
OP_TYPE_DIV,
OP_TYPE_MOD,
// TODO: check if below is necessary
#define TSDB_RELATION_INVALID 0
#define TSDB_RELATION_LESS 1
#define TSDB_RELATION_GREATER 2
#define TSDB_RELATION_EQUAL 3
#define TSDB_RELATION_LESS_EQUAL 4
#define TSDB_RELATION_GREATER_EQUAL 5
#define TSDB_RELATION_NOT_EQUAL 6
#define TSDB_RELATION_LIKE 7
#define TSDB_RELATION_NOT_LIKE 8
#define TSDB_RELATION_ISNULL 9
#define TSDB_RELATION_NOTNULL 10
#define TSDB_RELATION_IN 11
#define TSDB_RELATION_NOT_IN 12
// bit operator
OP_TYPE_BIT_AND,
OP_TYPE_BIT_OR,
#define TSDB_RELATION_AND 13
#define TSDB_RELATION_OR 14
#define TSDB_RELATION_NOT 15
// comparison operator
OP_TYPE_GREATER_THAN,
OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL,
OP_TYPE_EQUAL,
OP_TYPE_NOT_EQUAL,
OP_TYPE_IN,
OP_TYPE_NOT_IN,
OP_TYPE_LIKE,
OP_TYPE_NOT_LIKE,
OP_TYPE_MATCH,
OP_TYPE_NMATCH,
OP_TYPE_IS_NULL,
OP_TYPE_IS_NOT_NULL,
OP_TYPE_IS_TRUE,
OP_TYPE_IS_FALSE,
OP_TYPE_IS_UNKNOWN,
OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN,
#define TSDB_RELATION_MATCH 16
#define TSDB_RELATION_NMATCH 17
// json operator
OP_TYPE_JSON_GET_VALUE,
OP_TYPE_JSON_CONTAINS
} EOperatorType;
typedef enum ELogicConditionType {
LOGIC_COND_TYPE_AND,
LOGIC_COND_TYPE_OR,
LOGIC_COND_TYPE_NOT,
} ELogicConditionType;
#define TSDB_BINARY_OP_ADD 4000
#define TSDB_BINARY_OP_SUBTRACT 4001
#define TSDB_BINARY_OP_MULTIPLY 4002
#define TSDB_BINARY_OP_DIVIDE 4003
#define TSDB_BINARY_OP_REMAINDER 4004
#define TSDB_BINARY_OP_CONCAT 4005
#define FUNCTION_CEIL 4500
#define FUNCTION_FLOOR 4501
@ -149,9 +166,6 @@ do { \
#define FUNCTION_LTRIM 4802
#define FUNCTION_RTRIM 4803
#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
#define TSDB_NAME_DELIMITER_LEN 1
#define TSDB_UNI_LEN 24

2
source/common/src/tname.c
View File

@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil
}
assert(src->filterstr == 0 || src->filterstr == 1);
assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID));
assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0));
return pFilter;
}

2
source/common/src/ttypes.c
View File

@ -585,7 +585,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
}
void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
if (optr == TSDB_BINARY_OP_ADD) {
if (optr == OP_TYPE_ADD) {
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
*((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);

2
source/libs/catalog/src/catalog.c
View File

@ -546,7 +546,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
if (tbMeta->tableType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
}

1
source/libs/function/inc/builtins.h
View File

@ -46,6 +46,7 @@ typedef struct SBuiltinFuncDefinition {
FExecGetEnv getEnvFunc;
FExecInit initFunc;
FExecProcess processFunc;
FScalarExecProcess sprocessFunc;
FExecFinalize finalizeFunc;
} SBuiltinFuncDefinition;

2
source/libs/function/src/builtins.c
View File

@ -36,7 +36,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.processFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
}
};

8
source/libs/function/src/functionMgt.c
View File

@ -71,6 +71,14 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
return TSDB_CODE_SUCCESS;
}
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return TSDB_CODE_FAILED;
}
pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc;
return TSDB_CODE_SUCCESS;
}
bool fmIsAggFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;

11
source/libs/function/src/texpr.c
View File

@ -25,6 +25,7 @@
#include "thash.h"
#include "texpr.h"
#include "tvariant.h"
#include "tdef.h"
//static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
// if (pLeft->nodeType == TEXPR_COL_NODE) {
@ -94,7 +95,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp
//non-leaf nodes, recursively traverse the expression tree in the post-root order
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
if (pExpr->_node.optr == TSDB_RELATION_OR) { // or
if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) { // or
if (exprTreeApplyFilter(pLeft, pItem, param)) {
return true;
}
@ -255,7 +256,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) {
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_LIKE;
expr->_node.optr = OP_TYPE_LIKE;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1;
@ -266,7 +267,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_MATCH;
expr->_node.optr = OP_TYPE_MATCH;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1;
@ -276,7 +277,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) {
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_NMATCH;
expr->_node.optr = OP_TYPE_NMATCH;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_NMATCH_LEN) + 1;
@ -286,7 +287,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_IN;
expr->_node.optr = OP_TYPE_IN;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY;

56
source/libs/index/src/index.c
View File

@ -66,7 +66,9 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv);
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
pthread_once(&isInit, indexInit);
SIndex* sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; }
if (sIdx == NULL) {
return -1;
}
#ifdef USE_LUCENE
index_t* index = index_open(path);
@ -76,7 +78,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
#ifdef USE_INVERTED_INDEX
// sIdx->cache = (void*)indexCacheCreate(sIdx);
sIdx->tindex = indexTFileCreate(path);
if (sIdx->tindex == NULL) { goto END; }
if (sIdx->tindex == NULL) {
goto END;
}
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->cVersion = 1;
@ -87,7 +91,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
#endif
END:
if (sIdx != NULL) { indexClose(sIdx); }
if (sIdx != NULL) {
indexClose(sIdx);
}
*index = NULL;
return -1;
@ -103,7 +109,9 @@ void indexClose(SIndex* sIdx) {
void* iter = taosHashIterate(sIdx->colObj, NULL);
while (iter) {
IndexCache** pCache = iter;
if (*pCache) { indexCacheUnRef(*pCache); }
if (*pCache) {
indexCacheUnRef(*pCache);
}
iter = taosHashIterate(sIdx->colObj, iter);
}
taosHashCleanup(sIdx->colObj);
@ -161,7 +169,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
assert(*cache != NULL);
int ret = indexCachePut(*cache, p, uid);
if (ret != 0) { return ret; }
if (ret != 0) {
return ret;
}
}
#endif
@ -191,7 +201,9 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
int tsz = 0;
index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz);
for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); }
for (int i = 0; i < tsz; i++) {
taosArrayPush(result, &tResult[i]);
}
for (int i = 0; i < nQuery; i++) {
free(fields[i]);
@ -248,7 +260,9 @@ void indexOptsDestroy(SIndexOpts* opts) {
*/
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; }
if (p == NULL) {
return NULL;
}
p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p;
@ -270,7 +284,9 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
int32_t nColName, const char* colVal, int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { return NULL; }
if (t == NULL) {
return NULL;
}
t->suid = suid;
t->operType = oper;
@ -343,7 +359,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
return 0;
}
static void indexInterResultsDestroy(SArray* results) {
if (results == NULL) { return; }
if (results == NULL) {
return;
}
size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) {
@ -419,18 +437,24 @@ static void indexDestroyTempResult(SArray* result) {
taosArrayDestroy(result);
}
int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
if (sIdx == NULL) { return -1; }
if (sIdx == NULL) {
return -1;
}
indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
int64_t st = taosGetTimestampUs();
IndexCache* pCache = (IndexCache*)cache;
TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName);
if (pReader == NULL) { indexWarn("empty tfile reader found"); }
if (pReader == NULL) {
indexWarn("empty tfile reader found");
}
// handle flush
Iterate* cacheIter = indexCacheIteratorCreate(pCache);
Iterate* tfileIter = tfileIteratorCreate(pReader);
if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); }
if (tfileIter == NULL) {
indexWarn("empty tfile reader iterator");
}
SArray* result = taosArrayInit(1024, sizeof(void*));
@ -484,7 +508,9 @@ void iterateValueDestroy(IterateValue* value, bool destroy) {
taosArrayDestroy(value->val);
value->val = NULL;
} else {
if (value->val != NULL) { taosArrayClear(value->val); }
if (value->val != NULL) {
taosArrayClear(value->val);
}
}
free(value->colVal);
value->colVal = NULL;
@ -507,7 +533,9 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
tfileWriterClose(tw);
TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName);
if (reader == NULL) { return -1; }
if (reader == NULL) {
return -1;
}
TFileHeader* header = &reader->header;
ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)};

69
source/libs/index/src/index_cache.c
View File

@ -119,13 +119,17 @@ void indexCacheDestroySkiplist(SSkipList* slt) {
tSkipListDestroy(slt);
}
void indexCacheDestroyImm(IndexCache* cache) {
if (cache == NULL) { return; }
if (cache == NULL) {
return;
}
MemTable* tbl = NULL;
pthread_mutex_lock(&cache->mtx);
tbl = cache->imm;
cache->imm = NULL; // or throw int bg thread
pthread_cond_broadcast(&cache->finished);
pthread_mutex_unlock(&cache->mtx);
indexMemUnRef(tbl);
@ -133,7 +137,9 @@ void indexCacheDestroyImm(IndexCache* cache) {
}
void indexCacheDestroy(void* cache) {
IndexCache* pCache = cache;
if (pCache == NULL) { return; }
if (pCache == NULL) {
return;
}
indexMemUnRef(pCache->mem);
indexMemUnRef(pCache->imm);
free(pCache->colName);
@ -146,7 +152,9 @@ void indexCacheDestroy(void* cache) {
Iterate* indexCacheIteratorCreate(IndexCache* cache) {
Iterate* iiter = calloc(1, sizeof(Iterate));
if (iiter == NULL) { return NULL; }
if (iiter == NULL) {
return NULL;
}
pthread_mutex_lock(&cache->mtx);
@ -164,7 +172,9 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
return iiter;
}
void indexCacheIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; }
if (iter == NULL) {
return;
}
tSkipListDestroyIter(iter->iter);
iterateValueDestroy(&iter->val, true);
free(iter);
@ -186,9 +196,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
} else if (cache->imm != NULL) {
// TODO: wake up by condition variable
pthread_cond_wait(&cache->finished, &cache->mtx);
// pthread_mutex_unlock(&cache->mtx);
// taosMsleep(50);
// pthread_mutex_lock(&cache->mtx);
} else {
indexCacheRef(cache);
cache->imm = cache->mem;
@ -202,13 +209,17 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
}
int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
if (cache == NULL) { return -1; }
if (cache == NULL) {
return -1;
}
IndexCache* pCache = cache;
indexCacheRef(pCache);
// encode data
CacheTerm* ct = calloc(1, sizeof(CacheTerm));
if (cache == NULL) { return -1; }
if (cache == NULL) {
return -1;
}
// set up key
ct->colType = term->colType;
ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1));
@ -240,7 +251,9 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
}
static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) {
if (mem == NULL) { return 0; }
if (mem == NULL) {
return 0;
}
char* key = indexCacheTermGet(ct);
SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
@ -266,7 +279,9 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA
return 0;
}
int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) {
if (cache == NULL) { return 0; }
if (cache == NULL) {
return 0;
}
IndexCache* pCache = cache;
MemTable *mem = NULL, *imm = NULL;
@ -294,23 +309,33 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV
}
void indexCacheRef(IndexCache* cache) {
if (cache == NULL) { return; }
if (cache == NULL) {
return;
}
int ref = T_REF_INC(cache);
UNUSED(ref);
}
void indexCacheUnRef(IndexCache* cache) {
if (cache == NULL) { return; }
if (cache == NULL) {
return;
}
int ref = T_REF_DEC(cache);
if (ref == 0) { indexCacheDestroy(cache); }
if (ref == 0) {
indexCacheDestroy(cache);
}
}
void indexMemRef(MemTable* tbl) {
if (tbl == NULL) { return; }
if (tbl == NULL) {
return;
}
int ref = T_REF_INC(tbl);
UNUSED(ref);
}
void indexMemUnRef(MemTable* tbl) {
if (tbl == NULL) { return; }
if (tbl == NULL) {
return;
}
int ref = T_REF_DEC(tbl);
if (ref == 0) {
SSkipList* slt = tbl->mem;
@ -320,7 +345,9 @@ void indexMemUnRef(MemTable* tbl) {
}
static void indexCacheTermDestroy(CacheTerm* ct) {
if (ct == NULL) { return; }
if (ct == NULL) {
return;
}
free(ct->colVal);
free(ct);
}
@ -333,7 +360,9 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) {
CacheTerm* rt = (CacheTerm*)r;
// compare colVal
int32_t cmp = strcmp(lt->colVal, rt->colVal);
if (cmp == 0) { return rt->version - lt->version; }
if (cmp == 0) {
return rt->version - lt->version;
}
return cmp;
}
@ -354,7 +383,9 @@ static void doMergeWork(SSchedMsg* msg) {
}
static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListIterator* iter = itera->iter;
if (iter == NULL) { return false; }
if (iter == NULL) {
return false;
}
IterateValue* iv = &itera->val;
iterateValueDestroy(iv, false);

198
source/libs/index/src/index_fst.c
View File

@ -31,20 +31,24 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi
FstUnFinishedNodes* fstUnFinishedNodesCreate() {
FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes));
if (nodes == NULL) { return NULL; }
if (nodes == NULL) {
return NULL;
}
nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished));
fstUnFinishedNodesPushEmpty(nodes, false);
return nodes;
}
void unFinishedNodeDestroyElem(void* elem) {
static void unFinishedNodeDestroyElem(void* elem) {
FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem;
fstBuilderNodeDestroy(b->node);
free(b->last);
b->last = NULL;
}
void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) {
if (nodes == NULL) { return; }
if (nodes == NULL) {
return;
}
taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem);
free(nodes);
@ -92,7 +96,9 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr add
}
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) {
FstSlice* s = &bs;
if (fstSliceIsEmpty(s)) { return; }
if (fstSliceIsEmpty(s)) {
return;
}
size_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL);
@ -172,7 +178,9 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node,
FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
FstState fs = {.state = EmptyFinal, .val = 0};
if (addr == EMPTY_ADDRESS) { return fs; }
if (addr == EMPTY_ADDRESS) {
return fs;
}
uint8_t* data = fstSliceData(slice, NULL);
uint8_t v = data[addr];
@ -229,7 +237,9 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran
fstStateSetCommInput(&st, trn->inp);
bool null = false;
uint8_t inp = fstStateCommInput(&st, &null);
if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); }
if (null == true) {
fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp));
}
fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
return;
}
@ -263,7 +273,9 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
fstStateSetStateNtrans(&st, (uint8_t)sz);
if (anyOuts) {
if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); }
if (FST_BUILDER_NODE_IS_FINAL(node)) {
fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
}
for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition* t = taosArrayGet(node->trans, i);
fstCountingWriterPackUintIn(w, t->out, oSize);
@ -428,7 +440,9 @@ Output fstStateOutput(FstState* s, FstNode* node) {
assert(s->state == OneTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { return 0; }
if (oSizes == 0) {
return 0;
}
FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
@ -440,7 +454,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { return 0; }
if (oSizes == 0) {
return 0;
}
FstSlice* slice = &node->data;
uint8_t* data = fstSliceData(slice, NULL);
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -453,7 +469,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
void fstStateSetFinalState(FstState* s, bool yes) {
assert(s->state == AnyTrans);
if (yes) { s->val |= 0b01000000; }
if (yes) {
s->val |= 0b01000000;
}
return;
}
bool fstStateIsFinalState(FstState* s) {
@ -463,7 +481,9 @@ bool fstStateIsFinalState(FstState* s) {
void fstStateSetStateNtrans(FstState* s, uint8_t n) {
assert(s->state == AnyTrans);
if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; }
if (n <= 0b00111111) {
s->val = (s->val & 0b11000000) | n;
}
return;
}
// state_ntrans
@ -495,7 +515,9 @@ uint64_t fstStateNtransLen(FstState* s) {
uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
bool null = false;
uint8_t n = fstStateStateNtrans(s, &null);
if (null != true) { return n; }
if (null != true) {
return n;
}
int32_t len;
uint8_t* data = fstSliceData(slice, &len);
n = data[len - 2];
@ -505,7 +527,9 @@ uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
}
Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; }
if (oSizes == 0 || !fstStateIsFinalState(s)) {
return 0;
}
uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size
- fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes;
@ -522,7 +546,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
uint8_t* data = fstSliceData(slice, &dlen);
uint64_t i = data[at + b];
// uint64_t i = slice->data[slice->start + at + b];
if (i >= node->nTrans) { *null = true; }
if (i >= node->nTrans) {
*null = true;
}
return i;
} else {
uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size
@ -539,7 +565,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
return node->nTrans - i - 1; // bug
}
}
if (i == len) { *null = true; }
if (i == len) {
*null = true;
}
fstSliceDestroy(&t);
}
}
@ -548,7 +576,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) {
FstNode* n = (FstNode*)malloc(sizeof(FstNode));
if (n == NULL) { return NULL; }
if (n == NULL) {
return NULL;
}
FstState st = fstStateCreateFrom(slice, addr);
@ -614,7 +644,9 @@ void fstNodeDestroy(FstNode* node) {
}
FstTransitions* fstNodeTransitions(FstNode* node) {
FstTransitions* t = malloc(sizeof(FstTransitions));
if (NULL == t) { return NULL; }
if (NULL == t) {
return NULL;
}
FstRange range = {.start = 0, .end = FST_NODE_LEN(node)};
t->range = range;
t->node = node;
@ -721,7 +753,9 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled
FstBuilder* fstBuilderCreate(void* w, FstType ty) {
FstBuilder* b = malloc(sizeof(FstBuilder));
if (NULL == b) { return b; }
if (NULL == b) {
return b;
}
b->wrt = fstCountingWriterCreate(w);
b->unfinished = fstUnFinishedNodesCreate();
@ -735,15 +769,17 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
taosEncodeFixedU64(&pBuf64, VERSION);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
memset(buf64, 0, sizeof(buf64));
pBuf64 = buf64;
memset(buf64, 0, sizeof(buf64));
taosEncodeFixedU64(&pBuf64, ty);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
return b;
}
void fstBuilderDestroy(FstBuilder* b) {
if (b == NULL) { return; }
if (b == NULL) {
return;
}
fstCountingWriterDestroy(b->wrt);
fstUnFinishedNodesDestroy(b->unfinished);
@ -830,6 +866,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return;
}
CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
return EMPTY_ADDRESS;
@ -844,7 +881,9 @@ CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr);
b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1);
if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); }
if (entry->state == NOTFOUND) {
FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr);
}
fstRegistryEntryDestroy(entry);
return b->lastAddr;
@ -887,7 +926,9 @@ FstSlice fstNodeAsSlice(FstNode* node) {
FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
FstLastTransition* trn = malloc(sizeof(FstLastTransition));
if (trn == NULL) { return NULL; }
if (trn == NULL) {
return NULL;
}
trn->inp = inp;
trn->out = out;
@ -897,7 +938,9 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); }
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) {
FstLastTransition* trn = unNode->last;
if (trn == NULL) { return; }
if (trn == NULL) {
return;
}
FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
taosArrayPush(unNode->node->trans, &t);
fstLastTransitionDestroy(trn);
@ -906,27 +949,35 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp
}
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) {
if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; }
if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) {
unNode->node->finalOutput += out;
}
size_t sz = taosArrayGetSize(unNode->node->trans);
for (size_t i = 0; i < sz; i++) {
FstTransition* trn = taosArrayGet(unNode->node->trans, i);
trn->out += out;
}
if (unNode->last) { unNode->last->out += out; }
if (unNode->last) {
unNode->last->out += out;
}
return;
}
Fst* fstCreate(FstSlice* slice) {
int32_t slen;
char* buf = fstSliceData(slice, &slen);
if (slen < 36) { return NULL; }
if (slen < 36) {
return NULL;
}
uint64_t len = slen;
uint64_t skip = 0;
uint64_t version;
taosDecodeFixedU64(buf, &version);
skip += sizeof(version);
if (version == 0 || version > VERSION) { return NULL; }
if (version == 0 || version > VERSION) {
return NULL;
}
uint64_t type;
taosDecodeFixedU64(buf + skip, &type);
@ -949,10 +1000,14 @@ Fst* fstCreate(FstSlice* slice) {
taosDecodeFixedU64(buf + len, &fstLen);
// TODO(validate root addr)
Fst* fst = (Fst*)calloc(1, sizeof(Fst));
if (fst == NULL) { return NULL; }
if (fst == NULL) {
return NULL;
}
fst->meta = (FstMeta*)malloc(sizeof(FstMeta));
if (NULL == fst->meta) { goto FST_CREAT_FAILED; }
if (NULL == fst->meta) {
goto FST_CREAT_FAILED;
}
fst->meta->version = version;
fst->meta->rootAddr = rootAddr;
@ -983,7 +1038,7 @@ void fstDestroy(Fst* fst) {
bool fstGet(Fst* fst, FstSlice* b, Output* out) {
// dec lock range
pthread_mutex_lock(&fst->mtx);
// pthread_mutex_lock(&fst->mtx);
FstNode* root = fstGetRoot(fst);
Output tOut = 0;
int32_t len;
@ -996,7 +1051,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
uint8_t inp = data[i];
Output res = 0;
if (false == fstNodeFindInput(root, inp, &res)) {
pthread_mutex_unlock(&fst->mtx);
// pthread_mutex_unlock(&fst->mtx);
return false;
}
@ -1007,7 +1062,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
taosArrayPush(nodes, &root);
}
if (!FST_NODE_IS_FINAL(root)) {
pthread_mutex_unlock(&fst->mtx);
// pthread_mutex_unlock(&fst->mtx);
return false;
} else {
tOut = tOut + FST_NODE_FINAL_OUTPUT(root);
@ -1018,8 +1073,8 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
fstNodeDestroy(*node);
}
taosArrayDestroy(nodes);
fst->root = NULL;
pthread_mutex_unlock(&fst->mtx);
// fst->root = NULL;
// pthread_mutex_unlock(&fst->mtx);
*out = tOut;
return true;
}
@ -1028,7 +1083,9 @@ FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
return fstStreamBuilderCreate(fst, ctx);
}
StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
if (sb == NULL) { return NULL; }
if (sb == NULL) {
return NULL;
}
return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
}
FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
@ -1039,15 +1096,6 @@ FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
FstNode* fstGetRoot(Fst* fst) {
CompiledAddr rAddr = fstGetRootAddr(fst);
return fstGetNode(fst, rAddr);
// pthread_mutex_lock(&fst->mtx);
// if (fst->root != NULL) {
// // pthread_mutex_unlock(&fst->mtx);
// return fst->root;
//}
// CompiledAddr rAddr = fstGetRootAddr(fst);
// fst->root = fstGetNode(fst, rAddr);
//// pthread_mutex_unlock(&fst->mtx);
// return fst->root;
}
FstNode* fstGetNode(Fst* fst, CompiledAddr addr) {
@ -1074,14 +1122,18 @@ bool fstVerify(Fst* fst) {
uint32_t len, checkSum = fst->meta->checkSum;
uint8_t* data = fstSliceData(fst->data, &len);
TSCKSUM initSum = 0;
if (!taosCheckChecksumWhole(data, len)) { return false; }
if (!taosCheckChecksumWhole(data, len)) {
return false;
}
return true;
}
// data bound function
FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) {
FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData));
if (b == NULL) { return NULL; }
if (b == NULL) {
return NULL;
}
if (data != NULL) {
b->data = fstSliceCopy(data, data->start, data->end);
@ -1118,7 +1170,9 @@ void fstBoundDestroy(FstBoundWithData* bound) { free(bound); }
StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
FstBoundWithData* max) {
StreamWithState* sws = calloc(1, sizeof(StreamWithState));
if (sws == NULL) { return NULL; }
if (sws == NULL) {
return NULL;
}
sws->fst = fst;
sws->aut = automation;
@ -1134,7 +1188,9 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
return sws;
}
void streamWithStateDestroy(StreamWithState* sws) {
if (sws == NULL) { return; }
if (sws == NULL) {
return;
}
taosArrayDestroy(sws->inp);
taosArrayDestroyEx(sws->stack, streamStateDestroy);
@ -1200,7 +1256,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
uint64_t i = 0;
for (i = trans->range.start; i < trans->range.end; i++) {
FstTransition trn;
if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; }
if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) {
break;
}
}
StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
@ -1248,7 +1306,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
while (taosArrayGetSize(sws->stack) > 0) {
StreamState* p = (StreamState*)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) {
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); }
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
taosArrayPop(sws->inp);
}
streamStateDestroy(p);
continue;
}
@ -1267,7 +1327,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
if (FST_NODE_IS_FINAL(nextNode)) {
// void *eofState = sws->aut->acceptEof(nextState);
void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); }
if (eofState != NULL) {
isMatch = automFuncs[aut->type].isMatch(aut, eofState);
}
}
StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
taosArrayPush(sws->stack, &s1);
@ -1277,24 +1339,26 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
size_t isz = taosArrayGetSize(sws->inp);
uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t));
for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); }
for (uint32_t i = 0; i < isz; i++) {
buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i);
}
FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp));
if (fstBoundWithDataExceededBy(sws->endAt, &slice)) {
taosArrayDestroyEx(sws->stack, streamStateDestroy);
sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
free(buf);
tfree(buf);
fstSliceDestroy(&slice);
return NULL;
}
if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
free(buf);
tfree(buf);
fstSliceDestroy(&slice);
taosArrayDestroy(nodes);
return result;
}
free(buf);
tfree(buf);
fstSliceDestroy(&slice);
}
for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
@ -1307,16 +1371,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult));
if (result == NULL) { return NULL; }
if (result == NULL) {
return NULL;
}
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut;
result->state = state;
return result;
}
void swsResultDestroy(StreamWithStateResult* result) {
if (NULL == result) { return; }
if (NULL == result) {
return;
}
fstSliceDestroy(&result->data);
startWithStateValueDestroy(result->state);
@ -1324,16 +1391,18 @@ void swsResultDestroy(StreamWithStateResult* result) {
}
void streamStateDestroy(void* s) {
if (NULL == s) { return; }
if (NULL == s) {
return;
}
StreamState* ss = (StreamState*)s;
fstNodeDestroy(ss->node);
// free(s->autoState);
}
FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { return NULL; }
if (NULL == b) {
return NULL;
}
b->fst = fst;
b->aut = aut;
@ -1349,8 +1418,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) {
free(b);
}
FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
if (b == NULL) { return NULL; }
if (b == NULL) {
return NULL;
}
if (type == GE) {
b->min->type = Included;
fstSliceDestroy(&(b->min->data));

24
source/libs/index/src/index_fst_automation.c
View File

@ -17,7 +17,9 @@
StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; }
if (nsv == NULL) {
return NULL;
}
nsv->kind = kind;
nsv->type = ty;
@ -35,7 +37,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp
}
void startWithStateValueDestroy(void* val) {
StartWithStateValue* sv = (StartWithStateValue*)val;
if (sv == NULL) { return; }
if (sv == NULL) {
return;
}
if (sv->type == FST_INT) {
//
@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) {
}
StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; }
if (nsv == NULL) {
return NULL;
}
nsv->kind = sv->kind;
nsv->type = sv->type;
@ -88,10 +94,14 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
StartWithStateValue* ssv = (StartWithStateValue*)state;
if (ssv == NULL || ctx == NULL) { return NULL; }
if (ssv == NULL || ctx == NULL) {
return NULL;
}
char* data = ctx->data;
if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); }
if (ssv->kind == Done) {
return startWithStateValueCreate(Done, FST_INT, &ssv->val);
}
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1;
@ -128,7 +138,9 @@ AutomationFunc automFuncs[] = {
AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { return NULL; }
if (ctx == NULL) {
return NULL;
}
StartWithStateValue* sv = NULL;
if (atype == AUTOMATION_ALWAYS) {

25
source/libs/index/src/index_fst_util.c
View File

@ -29,18 +29,6 @@ const uint64_t VERSION = 3;
const uint64_t TRANS_INDEX_THRESHOLD = 32;
// uint8_t commonInput(uint8_t idx) {
// if (idx == 0) { return -1; }
// else {
// return COMMON_INPUTS_INV[idx - 1];
// }
//}
//
// uint8_t commonIdx(uint8_t v, uint8_t max) {
// uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256;
// return v > max ? 0: v;
//}
uint8_t packSize(uint64_t n) {
if (n < (1u << 8)) {
return 1;
@ -103,9 +91,6 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) {
FstString* str = s->str;
str->ref++;
// uint8_t *buf = fstSliceData(s, &alen);
// start = buf + start - (buf - s->start);
// end = buf + end - (buf - s->start);
FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
return t;
@ -130,19 +115,19 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
ans.end = tlen - 1;
return ans;
}
bool fstSliceIsEmpty(FstSlice* s) {
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
}
bool fstSliceIsEmpty(FstSlice* s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; }
uint8_t* fstSliceData(FstSlice* s, int32_t* size) {
FstString* str = s->str;
if (size != NULL) { *size = s->end - s->start + 1; }
if (size != NULL) {
*size = s->end - s->start + 1;
}
return str->data + s->start;
}
void fstSliceDestroy(FstSlice* s) {
FstString* str = s->str;
str->ref--;
if (str->ref <= 0) {
if (str->ref == 0) {
free(str->data);
free(str);
s->str = NULL;

173
source/libs/index/src/index_tfile.c
View File

@ -13,8 +13,6 @@ p *
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//#include <sys/types.h>
//#include <dirent.h>
#include "index_tfile.h"
#include "index.h"
#include "index_fst.h"
@ -61,7 +59,9 @@ static void tfileGenFileFullName(char* fullname, const char* path, uint64_t s
TFileCache* tfileCacheCreate(const char* path) {
TFileCache* tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; }
if (tcache == NULL) {
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64;
@ -98,7 +98,9 @@ End:
return NULL;
}
void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { return; }
if (tcache == NULL) {
return;
}
// free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
@ -119,7 +121,9 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
int32_t sz = indexSerialCacheKey(key, buf);
assert(sz < sizeof(buf));
TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
if (reader == NULL) { return NULL; }
if (reader == NULL) {
return NULL;
}
tfileReaderRef(*reader);
return *reader;
@ -142,7 +146,9 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
}
TFileReader* tfileReaderCreate(WriterCtx* ctx) {
TFileReader* reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) { return NULL; }
if (reader == NULL) {
return NULL;
}
reader->ctx = ctx;
@ -169,7 +175,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) {
return reader;
}
void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { return; }
if (reader == NULL) {
return;
}
// T_REF_INC(reader);
fstDestroy(reader->fst);
writerCtxDestroy(reader->ctx, reader->remove);
@ -209,7 +217,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c
tfileGenFileFullName(fullname, path, suid, colName, version);
// indexInfo("open write file name %s", fullname);
WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
if (wcx == NULL) { return NULL; }
if (wcx == NULL) {
return NULL;
}
TFileHeader tfh = {0};
tfh.suid = suid;
@ -225,7 +235,9 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size);
if (wc == NULL) { return NULL; }
if (wc == NULL) {
return NULL;
}
TFileReader* reader = tfileReaderCreate(wc);
return reader;
@ -316,19 +328,25 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
return 0;
}
void tfileWriterClose(TFileWriter* tw) {
if (tw == NULL) { return; }
if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx, false);
free(tw);
}
void tfileWriterDestroy(TFileWriter* tw) {
if (tw == NULL) { return; }
if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx, false);
free(tw);
}
IndexTFile* indexTFileCreate(const char* path) {
TFileCache* cache = tfileCacheCreate(path);
if (cache == NULL) { return NULL; }
if (cache == NULL) {
return NULL;
}
IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
if (tfile == NULL) {
@ -340,21 +358,27 @@ IndexTFile* indexTFileCreate(const char* path) {
return tfile;
}
void indexTFileDestroy(IndexTFile* tfile) {
if (tfile == NULL) { return; }
if (tfile == NULL) {
return;
}
tfileCacheDestroy(tfile->cache);
free(tfile);
}
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
int ret = -1;
if (tfile == NULL) { return ret; }
if (tfile == NULL) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile;
SIndexTerm* term = query->term;
ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
if (reader == NULL) { return 0; }
if (reader == NULL) {
return 0;
}
return tfileReaderSearch(reader, query, result);
}
@ -373,7 +397,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
TFileFstIter* tIter = iiter->iter;
StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
if (rt == NULL) { return false; }
if (rt == NULL) {
return false;
}
int32_t sz = 0;
char* ch = (char*)fstSliceData(&rt->data, &sz);
@ -383,7 +409,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
offset = (uint64_t)(rt->out.out);
swsResultDestroy(rt);
// set up iterate value
if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; }
if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) {
return false;
}
iv->colVal = colVal;
return true;
@ -394,7 +422,9 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter));
if (tIter == NULL) { return NULL; }
if (tIter == NULL) {
return NULL;
}
tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
tIter->fb = fstSearch(reader->fst, tIter->ctx);
@ -404,7 +434,9 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
}
Iterate* tfileIteratorCreate(TFileReader* reader) {
if (reader == NULL) { return NULL; }
if (reader == NULL) {
return NULL;
}
Iterate* iter = calloc(1, sizeof(Iterate));
iter->iter = tfileFstIteratorCreate(reader);
@ -419,7 +451,9 @@ Iterate* tfileIteratorCreate(TFileReader* reader) {
return iter;
}
void tfileIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; }
if (iter == NULL) {
return;
}
IterateValue* iv = &iter->val;
iterateValueDestroy(iv, true);
@ -434,7 +468,9 @@ void tfileIteratorDestroy(Iterate* iter) {
}
TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
if (tf == NULL) { return NULL; }
if (tf == NULL) {
return NULL;
}
ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
return tfileCacheGet(tf->cache, &key);
}
@ -446,7 +482,9 @@ static int tfileUidCompare(const void* a, const void* b) {
}
static int tfileStrCompare(const void* a, const void* b) {
int ret = strcmp((char*)a, (char*)b);
if (ret == 0) { return ret; }
if (ret == 0) {
return ret;
}
return ret < 0 ? -1 : 1;
}
@ -461,13 +499,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) {
TFileValue* tfileValueCreate(char* val) {
TFileValue* tf = calloc(1, sizeof(TFileValue));
if (tf == NULL) { return NULL; }
if (tf == NULL) {
return NULL;
}
tf->colVal = tstrdup(val);
tf->tableId = taosArrayInit(32, sizeof(uint64_t));
return tf;
}
int tfileValuePush(TFileValue* tf, uint64_t val) {
if (tf == NULL) { return -1; }
if (tf == NULL) {
return -1;
}
taosArrayPush(tf->tableId, &val);
return 0;
}
@ -489,7 +531,9 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
tw->header.fstOffset = fstOffset;
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; }
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) {
return -1;
}
indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx));
tw->offset += sizeof(fstOffset);
return 0;
@ -502,7 +546,9 @@ static int tfileWriteHeader(TFileWriter* writer) {
indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx));
int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
if (sizeof(buf) != nwrite) { return -1; }
if (sizeof(buf) != nwrite) {
return -1;
}
indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx));
writer->offset = nwrite;
@ -552,23 +598,23 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
return 0;
}
static int tfileReaderLoadFst(TFileReader* reader) {
// current load fst into memory, refactor it later
static int FST_MAX_SIZE = 64 * 1024 * 1024;
char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
if (buf == NULL) { return -1; }
WriterCtx* ctx = reader->ctx;
int size = ctx->size(ctx);
// current load fst into memory, refactor it later
int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber);
char* buf = calloc(1, fstSize);
if (buf == NULL) {
return -1;
}
int64_t ts = taosGetTimestampUs();
int32_t nread =
ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset);
int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset);
int64_t cost = taosGetTimestampUs() - ts;
indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost);
indexInfo("nread = %d, and fst offset=%d, size: %d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost);
// we assuse fst size less than FST_MAX_SIZE
assert(nread > 0 && nread < FST_MAX_SIZE);
assert(nread > 0 && nread <= fstSize);
FstSlice st = fstSliceCreate((uint8_t*)buf, nread);
reader->fst = fstCreate(&st);
@ -578,21 +624,35 @@ static int tfileReaderLoadFst(TFileReader* reader) {
return reader->fst != NULL ? 0 : -1;
}
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
int32_t nid;
// TODO(yihao): opt later
WriterCtx* ctx = reader->ctx;
char block[1024] = {0};
int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
assert(nread >= sizeof(uint32_t));
int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset);
assert(sizeof(nid) == nread);
char* p = block;
int32_t nid = *(int32_t*)p;
p += sizeof(nid);
int32_t total = sizeof(uint64_t) * nid;
char* buf = calloc(1, total);
if (buf == NULL) { return -1; }
while (nid > 0) {
int32_t left = block + sizeof(block) - p;
if (left >= sizeof(uint64_t)) {
taosArrayPush(result, (uint64_t*)p);
p += sizeof(uint64_t);
} else {
char buf[sizeof(uint64_t)] = {0};
memcpy(buf, p, left);
nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid));
assert(total == nread);
memset(block, 0, sizeof(block));
offset += sizeof(block);
nread = ctx->readFrom(ctx, block, sizeof(block), offset);
memcpy(buf + left, block, sizeof(uint64_t) - left);
for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); }
free(buf);
taosArrayPush(result, (uint64_t*)buf);
p = block + sizeof(uint64_t) - left;
}
nid -= 1;
}
return 0;
}
static int tfileReaderVerify(TFileReader* reader) {
@ -615,13 +675,17 @@ static int tfileReaderVerify(TFileReader* reader) {
}
void tfileReaderRef(TFileReader* reader) {
if (reader == NULL) { return; }
if (reader == NULL) {
return;
}
int ref = T_REF_INC(reader);
UNUSED(ref);
}
void tfileReaderUnRef(TFileReader* reader) {
if (reader == NULL) { return; }
if (reader == NULL) {
return;
}
int ref = T_REF_DEC(reader);
if (ref == 0) {
// do nothing
@ -630,18 +694,21 @@ void tfileReaderUnRef(TFileReader* reader) {
}
static SArray* tfileGetFileList(const char* path) {
SArray* files = taosArrayInit(4, sizeof(void*));
char buf[128] = {0};
uint64_t suid;
uint32_t version;
SArray* files = taosArrayInit(4, sizeof(void*));
DIR* dir = opendir(path);
if (NULL == dir) { return NULL; }
if (NULL == dir) {
return NULL;
}
struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
char* file = entry->d_name;
if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; }
if (0 != tfileParseFileName(file, &suid, buf, &version)) {
continue;
}
size_t len = strlen(path) + 1 + strlen(file) + 1;
char* buf = calloc(1, len);

20
source/libs/index/test/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_executable(indexTest "")
add_executable(fstTest "")
add_executable(fstUT "")
target_sources(indexTest
PRIVATE
"indexTests.cc"
@ -8,6 +10,11 @@ target_sources(fstTest
PRIVATE
"fstTest.cc"
)
target_sources(fstUT
PRIVATE
"fstUT.cc"
)
target_include_directories ( indexTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
@ -18,6 +25,12 @@ target_include_directories ( fstTest
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories ( fstUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest
os
util
@ -32,6 +45,13 @@ target_link_libraries (fstTest
gtest_main
index
)
target_link_libraries (fstUT
os
util
common
gtest_main
index
)
#add_test(

11
source/libs/index/test/fstTest.cc
View File

@ -58,7 +58,9 @@ class FstReadMemory {
bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) { return false; }
if (nRead <= 0) {
return false;
}
_size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s);
@ -97,7 +99,8 @@ class FstReadMemory {
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
swsResultDestroy(rt);
}
for (size_t i = 0; i < result.size(); i++) {}
for (size_t i = 0; i < result.size(); i++) {
}
std::cout << std::endl;
return true;
}
@ -173,7 +176,9 @@ void checkMillonWriteAndReadOfFst() {
delete fw;
FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
if (fr->init()) { printf("success to init fst read"); }
if (fr->init()) {
printf("success to init fst read");
}
Performance_fstReadRecords(fr);
tfCleanup();

233
source/libs/index/test/fstUT.cc Normal file
View File

@ -0,0 +1,233 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
#include "ulog.h"
static std::string dir = "/tmp/index";
static char indexlog[PATH_MAX] = {0};
static char tindex[PATH_MAX] = {0};
static char tindexDir[PATH_MAX] = {0};
static void EnvInit() {
tfInit();
std::string path = dir;
taosRemoveDir(path.c_str());
taosMkDir(path.c_str());
// init log file
snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str());
if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) {
printf("failed to init log");
}
// init index file
memset(tindex, 0, sizeof(tindex));
snprintf(tindex, PATH_MAX, "%s/tindex.idx", path.c_str());
}
static void EnvCleanup() {}
class FstWriter {
public:
FstWriter() {
_wc = writerCtxCreate(TFile, tindex, false, 64 * 1024 * 1024);
_b = fstBuilderCreate(_wc, 0);
}
bool Put(const std::string& key, uint64_t val) {
// char buf[128] = {0};
// int len = 0;
// taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
// FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
~FstWriter() {
fstBuilderFinish(_b);
fstBuilderDestroy(_b);
writerCtxDestroy(_wc, false);
}
private:
FstBuilder* _b;
WriterCtx* _wc;
};
class FstReadMemory {
public:
FstReadMemory(size_t size) {
_wc = writerCtxCreate(TFile, tindex, true, 64 * 1024);
_w = fstCountingWriterCreate(_wc);
_size = size;
memset((void*)&_s, 0, sizeof(_s));
}
bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) {
return false;
}
_size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s);
free(buf);
return _fst != NULL;
}
bool Get(const std::string& key, uint64_t* val) {
// char buf[128] = {0};
// int len = 0;
// taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
// FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
int64_t e = taosGetTimestampUs();
*elapse = e - s;
return ok;
}
// add later
bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
FstStreamBuilder* sb = fstSearch(_fst, ctx);
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
// result.push_back((uint64_t)(rt->out.out));
FstSlice* s = &rt->data;
int32_t sz = 0;
char* ch = (char*)fstSliceData(s, &sz);
std::string key(ch, sz);
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
swsResultDestroy(rt);
}
for (size_t i = 0; i < result.size(); i++) {
}
std::cout << std::endl;
return true;
}
bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() {
fstCountingWriterDestroy(_w);
fstDestroy(_fst);
fstSliceDestroy(&_s);
writerCtxDestroy(_wc, false);
tfCleanup();
}
private:
FstCountingWriter* _w;
Fst* _fst;
FstSlice _s;
WriterCtx* _wc;
size_t _size;
};
class FstWriterEnv : public ::testing::Test {
protected:
virtual void SetUp() { fw = new FstWriter(); }
virtual void TearDown() { delete fw; }
FstWriter* fw = NULL;
};
class FstReadEnv : public ::testing::Test {
protected:
virtual void SetUp() { fr = new FstReadMemory(1024); }
virtual void TearDown() { delete fr; }
FstReadMemory* fr = NULL;
};
class TFst {
public:
void CreateWriter() { fw = new FstWriter; }
void ReCreateWriter() {
if (fw != NULL) delete fw;
fw = new FstWriter;
}
void DestroyWriter() {
if (fw != NULL) delete fw;
}
void CreateReader() {
fr = new FstReadMemory(1024);
fr->init();
}
void ReCreateReader() {
if (fr != NULL) delete fr;
fr = new FstReadMemory(1024);
}
void DestroyReader() {
delete fr;
fr = NULL;
}
bool Put(const std::string& k, uint64_t v) {
if (fw == NULL) {
return false;
}
return fw->Put(k, v);
}
bool Get(const std::string& k, uint64_t* v) {
if (fr == NULL) {
return false;
}
return fr->Get(k, v);
}
private:
FstWriter* fw;
FstReadMemory* fr;
};
class FstEnv : public ::testing::Test {
protected:
virtual void SetUp() {
EnvInit();
fst = new TFst;
}
virtual void TearDown() { delete fst; }
TFst* fst;
};
TEST_F(FstEnv, writeNormal) {
fst->CreateWriter();
std::string str("aa");
for (int i = 0; i < 10; i++) {
str[0] = 'a' + i;
str.resize(2);
assert(fst->Put(str, i) == true);
}
// order failed
assert(fst->Put("aa", 1) == false);
fst->DestroyWriter();
fst->CreateReader();
uint64_t val;
assert(fst->Get("a", &val) == false);
assert(fst->Get("aa", &val) == true);
assert(val == 0);
}
TEST_F(FstEnv, writeExcpet) {}

19
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -142,6 +142,21 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
return (SNode*)pDst;
}
static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) {
dataTypeCopy(&pSrc->dataType, &pDst->dataType);
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
COPY_SCALAR_FIELD(columnId);
return (SNode*)pDst;
}
static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
COPY_NODE_FIELD(pExpr);
return (SNode*)pDst;
}
static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) {
COPY_SCALAR_FIELD(groupingSetType);
COPY_NODE_LIST_FIELD(pParameterList);
@ -168,6 +183,10 @@ SNode* nodesCloneNode(const SNode* pNode) {
return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst);
case QUERY_NODE_FUNCTION:
return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst);
case QUERY_NODE_COLUMN_REF:
return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst);
case QUERY_NODE_TARGET:
return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst);
case QUERY_NODE_REAL_TABLE:
case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE:

201
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -61,6 +61,10 @@ static char* nodeName(ENodeType type) {
return "Target";
case QUERY_NODE_RAW_EXPR:
return "RawExpr";
case QUERY_NODE_TUPLE_DESC:
return "TupleDesc";
case QUERY_NODE_SLOT_DESC:
return "SlotDesc";
case QUERY_NODE_SET_OPERATOR:
return "SetOperator";
case QUERY_NODE_SELECT_STMT:
@ -71,16 +75,22 @@ static char* nodeName(ENodeType type) {
return "LogicScan";
case QUERY_NODE_LOGIC_PLAN_JOIN:
return "LogicJoin";
case QUERY_NODE_LOGIC_PLAN_FILTER:
return "LogicFilter";
case QUERY_NODE_LOGIC_PLAN_AGG:
return "LogicAgg";
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return "LogicProject";
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return "PhysiTagScan";
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return "PhysiTableScan";
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return "PhysiProject";
default:
break;
}
return "Unknown";
static char tmp[20];
snprintf(tmp, sizeof(tmp), "Unknown %d", type);
return tmp;
}
static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const SNodeList* pList) {
@ -183,8 +193,93 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t logicFilterNodeToJson(const void* pObj, SJson* pJson) {
return logicPlanNodeToJson(pObj, pJson);
static const char* jkPhysiPlanOutputTuple = "OutputTuple";
static const char* jkPhysiPlanConditions = "Conditions";
static const char* jkPhysiPlanChildren = "Children";
static int32_t physicPlanNodeToJson(const void* pObj, SJson* pJson) {
const SPhysiNode* pNode = (const SPhysiNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputTuple, nodeToJson, &pNode->outputTuple);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkPhysiPlanConditions, nodeToJson, pNode->pConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkPhysiPlanChildren, nodeToJson, pNode->pChildren);
}
return code;
}
static const char* jkScanPhysiPlanScanCols = "ScanCols";
static const char* jkScanPhysiPlanTableId = "TableId";
static const char* jkScanPhysiPlanTableType = "TableType";
static const char* jkScanPhysiPlanScanOrder = "ScanOrder";
static const char* jkScanPhysiPlanScanCount = "ScanCount";
static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount";
static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkScanPhysiPlanScanCols, nodeToJson, pNode->pScanCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse);
}
return code;
}
static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) {
return physiScanNodeToJson(pObj, pJson);
}
static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag";
static const char* jkTableScanPhysiPlanStartKey = "StartKey";
static const char* jkTableScanPhysiPlanEndKey = "EndKey";
static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
int32_t code = physiScanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanEndKey, pNode->scanRange.ekey);
}
return code;
}
static const char* jkProjectPhysiPlanProjections = "Projections";
static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkProjectPhysiPlanProjections, nodeToJson, pNode->pProjections);
}
return code;
}
static const char* jkAggLogicPlanGroupKeys = "GroupKeys";
@ -277,19 +372,6 @@ static int32_t columnNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
// typedef struct SValueNode {
// SExprNode node; // QUERY_NODE_VALUE
// char* ;
// bool ;
// union {
// bool b;
// int64_t i;
// uint64_t u;
// double d;
// char* p;
// } datum;
// } SValueNode;
static const char* jkValueLiteral = "Literal";
static const char* jkValueDuration = "Duration";
static const char* jkValueDatum = "Datum";
@ -421,6 +503,74 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static const char* jkColumnRefDataType = "DataType";
static const char* jkColumnRefTupleId = "TupleId";
static const char* jkColumnRefSlotId = "SlotId";
static const char* jkColumnRefColumnId = "ColumnId";
static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) {
const SColumnRefNode* pNode = (const SColumnRefNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId);
}
return code;
}
static const char* jkTargetTupleId = "TupleId";
static const char* jkTargetSlotId = "SlotId";
static const char* jkTargetExpr = "Expr";
static int32_t targetNodeToJson(const void* pObj, SJson* pJson) {
const STargetNode* pNode = (const STargetNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkTargetTupleId, pNode->tupleId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTargetSlotId, pNode->slotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTargetExpr, nodeToJson, pNode->pExpr);
}
return code;
}
static const char* jkSlotDescSlotId = "SlotId";
static const char* jkSlotDescDataType = "DataType";
static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
const SSlotDescNode* pNode = (const SSlotDescNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSlotDescSlotId, pNode->slotId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSlotDescDataType, dataTypeToJson, &pNode->dataType);
}
return code;
}
static const char* jkTupleDescTupleId = "TupleId";
static const char* jkTupleDescSlots = "Slots";
static int32_t tupleDescNodeToJson(const void* pObj, SJson* pJson) {
const STupleDescNode* pNode = (const STupleDescNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkTupleDescTupleId, pNode->tupleId);
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkTupleDescSlots, nodeToJson, pNode->pSlots);
}
return code;
}
static const char* jkSelectStmtDistinct = "Distinct";
static const char* jkSelectStmtProjections = "Projections";
static const char* jkSelectStmtFrom = "From";
@ -497,8 +647,15 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_NODE_LIST:
case QUERY_NODE_FILL:
case QUERY_NODE_COLUMN_REF:
return columnRefNodeToJson(pObj, pJson);
case QUERY_NODE_TARGET:
return targetNodeToJson(pObj, pJson);
case QUERY_NODE_RAW_EXPR:
break;
case QUERY_NODE_TUPLE_DESC:
return tupleDescNodeToJson(pObj, pJson);
case QUERY_NODE_SLOT_DESC:
return slotDescNodeToJson(pObj, pJson);
case QUERY_NODE_SET_OPERATOR:
break;
case QUERY_NODE_SELECT_STMT:
@ -509,12 +666,16 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return logicScanNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_JOIN:
return logicJoinNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_FILTER:
return logicFilterNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_AGG:
return logicAggNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return logicProjectNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return physiTableScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return physiProjectNodeToJson(pObj, pJson);
default:
break;
}

28
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -63,6 +63,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SNodeListNode));
case QUERY_NODE_FILL:
return makeNode(type, sizeof(SFillNode));
case QUERY_NODE_COLUMN_REF:
return makeNode(type, sizeof(SColumnRefNode));
case QUERY_NODE_RAW_EXPR:
return makeNode(type, sizeof(SRawExprNode));
case QUERY_NODE_SET_OPERATOR:
@ -75,12 +77,22 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SScanLogicNode));
case QUERY_NODE_LOGIC_PLAN_JOIN:
return makeNode(type, sizeof(SJoinLogicNode));
case QUERY_NODE_LOGIC_PLAN_FILTER:
return makeNode(type, sizeof(SFilterLogicNode));
case QUERY_NODE_LOGIC_PLAN_AGG:
return makeNode(type, sizeof(SAggLogicNode));
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectLogicNode));
case QUERY_NODE_TARGET:
return makeNode(type, sizeof(STargetNode));
case QUERY_NODE_TUPLE_DESC:
return makeNode(type, sizeof(STupleDescNode));
case QUERY_NODE_SLOT_DESC:
return makeNode(type, sizeof(SSlotDescNode));
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return makeNode(type, sizeof(STagScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return makeNode(type, sizeof(STableScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode));
default:
break;
}
@ -184,29 +196,29 @@ void nodesDestroyList(SNodeList* pList) {
tfree(pList);
}
void *nodesGetValueFromNode(SValueNode *pNode) {
void* nodesGetValueFromNode(SValueNode *pNode) {
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_BOOL:
return (void *)&pNode->datum.b;
return (void*)&pNode->datum.b;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
return (void *)&pNode->datum.i;
return (void*)&pNode->datum.i;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
return (void *)&pNode->datum.u;
return (void*)&pNode->datum.u;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
return (void *)&pNode->datum.d;
return (void*)&pNode->datum.d;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
return (void *)pNode->datum.p;
return (void*)pNode->datum.p;
default:
break;
}

10
source/libs/parser/src/parserImpl.c
View File

@ -500,22 +500,24 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) {
}
static int32_t trimStringCopy(const char* src, int32_t len, char* dst) {
varDataSetLen(dst, len);
char* dstVal = varDataVal(dst);
// delete escape character: \\, \', \"
char delim = src[0];
int32_t cnt = 0;
int32_t j = 0;
for (uint32_t k = 1; k < len - 1; ++k) {
if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) {
dst[j] = src[k + 1];
dstVal[j] = src[k + 1];
cnt++;
j++;
k++;
continue;
}
dst[j] = src[k];
dstVal[j] = src[k];
j++;
}
dst[j] = '\0';
dstVal[j] = '\0';
return j;
}
@ -560,7 +562,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY: {
int32_t n = strlen(pVal->literal);
pVal->datum.p = calloc(1, n);
pVal->datum.p = calloc(1, n + VARSTR_HEADER_SIZE);
if (NULL == pVal->datum.p) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
return DEAL_RES_ERROR;

40
source/libs/parser/src/parserUtil.c
View File

@ -884,7 +884,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil
}
assert(src->filterstr == 0 || src->filterstr == 1);
assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID));
assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0));
return pFilter;
}
@ -1507,45 +1507,45 @@ int32_t getTagFilterSerializeLen(SQueryStmtInfo* pQueryInfo) {
uint32_t convertRelationalOperator(SToken *pToken) {
switch (pToken->type) {
case TK_LT:
return TSDB_RELATION_LESS;
return OP_TYPE_LOWER_THAN;
case TK_LE:
return TSDB_RELATION_LESS_EQUAL;
return OP_TYPE_LOWER_EQUAL;
case TK_GT:
return TSDB_RELATION_GREATER;
return OP_TYPE_GREATER_THAN;
case TK_GE:
return TSDB_RELATION_GREATER_EQUAL;
return OP_TYPE_GREATER_EQUAL;
case TK_NE:
return TSDB_RELATION_NOT_EQUAL;
return OP_TYPE_NOT_EQUAL;
case TK_AND:
return TSDB_RELATION_AND;
return LOGIC_COND_TYPE_AND;
case TK_OR:
return TSDB_RELATION_OR;
return LOGIC_COND_TYPE_OR;
case TK_EQ:
return TSDB_RELATION_EQUAL;
return OP_TYPE_EQUAL;
case TK_PLUS:
return TSDB_BINARY_OP_ADD;
return OP_TYPE_ADD;
case TK_MINUS:
return TSDB_BINARY_OP_SUBTRACT;
return OP_TYPE_SUB;
case TK_STAR:
return TSDB_BINARY_OP_MULTIPLY;
return OP_TYPE_MULTI;
case TK_SLASH:
case TK_DIVIDE:
return TSDB_BINARY_OP_DIVIDE;
return OP_TYPE_DIV;
case TK_REM:
return TSDB_BINARY_OP_REMAINDER;
return OP_TYPE_MOD;
case TK_LIKE:
return TSDB_RELATION_LIKE;
return OP_TYPE_LIKE;
case TK_MATCH:
return TSDB_RELATION_MATCH;
return OP_TYPE_MATCH;
case TK_NMATCH:
return TSDB_RELATION_NMATCH;
return OP_TYPE_NMATCH;
case TK_ISNULL:
return TSDB_RELATION_ISNULL;
return OP_TYPE_IS_NULL;
case TK_NOTNULL:
return TSDB_RELATION_NOTNULL;
return OP_TYPE_IS_NOT_NULL;
case TK_IN:
return TSDB_RELATION_IN;
return OP_TYPE_IN;
default: { return 0; }
}
}

2
source/libs/parser/test/parserTests.cpp
View File

@ -593,7 +593,7 @@ TEST(testCase, function_Test6) {
SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pQueryInfo->exprList[1], 0);
ASSERT_EQ(p2->pExpr->nodeType, TEXPR_BINARYEXPR_NODE);
ASSERT_EQ(p2->pExpr->_node.optr, TSDB_BINARY_OP_ADD);
ASSERT_EQ(p2->pExpr->_node.optr, OP_TYPE_ADD);
ASSERT_EQ(p2->pExpr->_node.pLeft->nodeType, TEXPR_COL_NODE);
ASSERT_EQ(p2->pExpr->_node.pRight->nodeType, TEXPR_COL_NODE);

1
source/libs/planner/inc/plannerImpl.h
View File

@ -24,6 +24,7 @@ extern "C" {
#include "planner.h"
int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode);
int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode);
#ifdef __cplusplus
}

351
source/libs/planner/src/plannerImpl.c
View File

@ -19,7 +19,6 @@
#define CHECK_ALLOC(p, res) \
do { \
if (NULL == (p)) { \
printf("%s : %d\n", __FUNCTION__, __LINE__); \
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \
return (res); \
} \
@ -29,7 +28,6 @@
do { \
int32_t code = (exec); \
if (TSDB_CODE_SUCCESS != code) { \
printf("%s : %d\n", __FUNCTION__, __LINE__); \
pCxt->errCode = code; \
return (res); \
} \
@ -38,7 +36,6 @@
typedef struct SPlanContext {
int32_t errCode;
int32_t planNodeId;
SNodeList* pResource;
} SPlanContext;
static SLogicNode* createQueryLogicNode(SPlanContext* pCxt, SNode* pStmt);
@ -60,10 +57,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
FOREACH(pExpr, pCxt->pExprs) {
if (nodesEqualNode(pExpr, *pNode)) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
CHECK_ALLOC(pCol, DEAL_RES_ERROR);
SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode);
pCol->node.resType = pToBeRewrittenExpr->resType;
strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName);
@ -222,26 +216,6 @@ static SLogicNode* createLogicNodeByTable(SPlanContext* pCxt, SSelectStmt* pSele
return NULL;
}
static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SLogicNode* pChild, SSelectStmt* pSelect) {
if (NULL == pSelect->pWhere) {
return NULL;
}
SFilterLogicNode* pFilter = (SFilterLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILTER);
CHECK_ALLOC(pFilter, NULL);
pFilter->node.id = pCxt->planNodeId++;
// set filter conditions
pFilter->node.pConditions = nodesCloneNode(pSelect->pWhere);
CHECK_ALLOC(pFilter->node.pConditions, (SLogicNode*)pFilter);
// set the output
pFilter->node.pTargets = nodesCloneList(pChild->pTargets);
CHECK_ALLOC(pFilter->node.pTargets, (SLogicNode*)pFilter);
return (SLogicNode*)pFilter;
}
typedef struct SCreateColumnCxt {
int32_t errCode;
SNodeList* pList;
@ -252,10 +226,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN: {
SNode* pCol = nodesCloneNode(pNode);
if (NULL == pCol || TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, pCol)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
CHECK_ALLOC(pCol, DEAL_RES_ERROR);
CHECK_CODE(nodesListAppend(pCxt->pList, pCol), DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
}
case QUERY_NODE_OPERATOR:
@ -263,16 +235,10 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
case QUERY_NODE_FUNCTION: {
SExprNode* pExpr = (SExprNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
CHECK_ALLOC(pCol, DEAL_RES_ERROR);
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
if (TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, (SNode*)pCol)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
CHECK_CODE(nodesListAppend(pCxt->pList, (SNode*)pCol), DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
}
default:
@ -284,9 +250,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) {
SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() };
if (NULL == cxt.pList) {
return NULL;
}
CHECK_ALLOC(cxt.pList, NULL);
nodesWalkList(pExprs, doCreateColumn, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pList);
@ -379,8 +344,9 @@ static SLogicNode* createProjectLogicNode(SPlanContext* pCxt, SSelectStmt* pSele
static SLogicNode* createSelectLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) {
SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createWhereFilterLogicNode(pCxt, pRoot, pSelect));
if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) {
pRoot->pConditions = nodesCloneNode(pSelect->pWhere);
CHECK_ALLOC(pRoot->pConditions, pRoot);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect));
@ -410,3 +376,300 @@ int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) {
*pLogicNode = pRoot;
return TSDB_CODE_SUCCESS;
}
int32_t optimize(SLogicNode* pLogicNode) {
// todo
return TSDB_CODE_SUCCESS;
}
typedef struct SSubLogicPlan {
SNode* pRoot; // SLogicNode
bool haveSuperTable;
bool haveSystemTable;
} SSubLogicPlan;
int32_t splitLogicPlan(SSubLogicPlan* pLogicPlan) {
// todo
return TSDB_CODE_SUCCESS;
}
typedef struct SSlotIndex {
int16_t tupleId;
int16_t slotId;
} SSlotIndex;
typedef struct SPhysiPlanContext {
int32_t errCode;
int16_t nextTupleId;
SArray* pTupleHelper;
} SPhysiPlanContext;
static int32_t getSlotKey(SNode* pNode, char* pKey) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName);
} else {
return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
}
}
static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) {
SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
if (NULL == pCol) {
return NULL;
}
pCol->dataType = ((SExprNode*)pNode)->resType;
pCol->tupleId = tupleId;
pCol->slotId = slotId;
pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1);
return (SNode*)pCol;
}
static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) {
SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC);
CHECK_ALLOC(pSlot, NULL);
pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->srcTupleId = -1;
pSlot->srcSlotId = -1;
pSlot->reserve = false;
pSlot->output = true;
return (SNode*)pSlot;
}
static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) {
STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET);
if (NULL == pTarget) {
return NULL;
}
pTarget->tupleId = tupleId;
pTarget->slotId = slotId;
pTarget->pExpr = nodesCloneNode(pNode);
if (NULL == pTarget->pExpr) {
nodesDestroyNode((SNode*)pTarget);
return NULL;
}
return (SNode*)pTarget;
}
static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) {
pTuple->tupleId = pCxt->nextTupleId++;
SHashObj* pHash = NULL;
if (NULL == pTuple->pSlots) {
pTuple->pSlots = nodesMakeList();
CHECK_ALLOC(pTuple->pSlots, TSDB_CODE_OUT_OF_MEMORY);
pHash = taosHashInit(LIST_LENGTH(pList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY);
if (NULL == taosArrayInsert(pCxt->pTupleHelper, pTuple->tupleId, &pHash)) {
taosHashCleanup(pHash);
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
}
*pOutput = nodesMakeList();
CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY);
SNode* pNode = NULL;
int16_t slotId = 0;
FOREACH(pNode, pList) {
SNode* pSlot = createSlotDesc(pCxt, pNode, slotId);
CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY);
if (TSDB_CODE_SUCCESS != nodesListAppend(pTuple->pSlots, (SNode*)pSlot)) {
nodesDestroyNode(pSlot);
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) {
nodesDestroyNode(pTarget);
return TSDB_CODE_OUT_OF_MEMORY;
}
SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId };
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(pNode, name);
CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY);
++slotId;
}
return TSDB_CODE_SUCCESS;
}
typedef struct STransformCxt {
int32_t errCode;
SHashObj* pHash;
} STransformCxt;
static EDealRes doTransform(SNode** pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
STransformCxt* pCxt = (STransformCxt*)pContext;
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(*pNode, name);
SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len);
if (NULL != pIndex) {
*pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId);
CHECK_ALLOC(*pNode, DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
}
}
return DEAL_RES_CONTINUE;
}
static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) {
SNode* pRes = nodesCloneNode(pNode);
CHECK_ALLOC(pRes, NULL);
STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) };
nodesRewriteNode(&pRes, doTransform, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode(pRes);
return NULL;
}
return pRes;
}
static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) {
SNodeList* pRes = nodesCloneList(pList);
CHECK_ALLOC(pRes, NULL);
STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) };
nodesRewriteList(pRes, doTransform, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(pRes);
return NULL;
}
return pRes;
}
static SPhysiNode* makePhysiNode(ENodeType type) {
SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type);
if (NULL == pPhysiNode) {
return NULL;
}
pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC;
return pPhysiNode;
}
static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY);
if (NULL != pScanLogicNode->node.pConditions) {
pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions);
CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY);
}
pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
pScanPhysiNode->order = TSDB_ORDER_ASC;
pScanPhysiNode->count = 1;
pScanPhysiNode->reverse = 0;
return TSDB_CODE_SUCCESS;
}
static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
CHECK_ALLOC(pTagScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan);
return (SPhysiNode*)pTagScan;
}
static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
pTableScan->scanFlag = pScanLogicNode->scanFlag;
pTableScan->scanRange = pScanLogicNode->scanRange;
return (SPhysiNode*)pTableScan;
}
static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pScanLogicNode);
case SCAN_TYPE_TABLE:
return createTableScanPhysiNode(pCxt, pScanLogicNode);
case SCAN_TYPE_STABLE:
case SCAN_TYPE_STREAM:
break;
default:
break;
}
}
static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL);
SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections);
CHECK_ALLOC(pProjections, (SPhysiNode*)pProject);
CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject);
nodesDestroyList(pProjections);
if (NULL != pProjectLogicNode->node.pConditions) {
pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions);
CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject);
}
return (SPhysiNode*)pProject;
}
static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) {
SNodeList* pChildern = nodesMakeList();
CHECK_ALLOC(pChildern, NULL);
SNode* pLogicChild;
FOREACH(pLogicChild, pLogicPlan->pChildren) {
SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild);
if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
nodesDestroyList(pChildern);
return NULL;
}
}
SPhysiNode* pPhyNode = NULL;
switch (nodeType(pLogicPlan)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan);
break;
case QUERY_NODE_LOGIC_PLAN_JOIN:
break;
case QUERY_NODE_LOGIC_PLAN_AGG:
break;
case QUERY_NODE_LOGIC_PLAN_PROJECT:
pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan);
break;
default:
break;
}
if (NULL != pPhyNode) {
pPhyNode->pChildren = pChildern;
SNode* pChild;
FOREACH(pChild, pPhyNode->pChildren) {
((SPhysiNode*)pChild)->pParent = pPhyNode;
}
}
return pPhyNode;
}
int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) {
SPhysiPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .nextTupleId = 0, .pTupleHelper = taosArrayInit(32, POINTER_BYTES) };
if (NULL == cxt.pTupleHelper) {
return TSDB_CODE_OUT_OF_MEMORY;
}
*pPhyNode = createPhysiNode(&cxt, pLogicNode);
return cxt.errCode;
}
int32_t buildPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) {
// split
// scale out
// maping
// create
return TSDB_CODE_SUCCESS;
}

30
source/libs/planner/test/newPlannerTest.cpp
View File

@ -25,6 +25,11 @@ using namespace testing;
class NewPlannerTest : public Test {
protected:
enum TestTarget {
TEST_LOGIC_PLAN,
TEST_PHYSICAL_PLAN
};
void setDatabase(const string& acctId, const string& db) {
acctId_ = acctId;
db_ = db;
@ -40,7 +45,7 @@ protected:
cxt_.pSql = sqlBuf_.c_str();
}
bool run() {
bool run(TestTarget target = TEST_PHYSICAL_PLAN) {
int32_t code = parser(&cxt_, &query_);
if (code != TSDB_CODE_SUCCESS) {
@ -53,17 +58,27 @@ protected:
SLogicNode* pLogicPlan = nullptr;
code = createLogicPlan(query_.pRoot, &pLogicPlan);
if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] plan code:" << code << ", strerror:" << tstrerror(code) << endl;
cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl;
return false;
}
cout << "sql : [" << cxt_.pSql << "]" << endl;
cout << "syntax test : " << endl;
cout << syntaxTreeStr << endl;
// cout << "logic plan : " << endl;
// cout << toString((const SNode*)pLogicPlan) << endl;
cout << "unformatted logic plan : " << endl;
cout << toString((const SNode*)pLogicPlan, false) << endl;
if (TEST_PHYSICAL_PLAN == target) {
SPhysiNode* pPhyPlan = nullptr;
code = createPhysiPlan(pLogicPlan, &pPhyPlan);
if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] physical plan code:" << code << ", strerror:" << tstrerror(code) << endl;
return false;
}
cout << "unformatted physical plan : " << endl;
cout << toString((const SNode*)pPhyPlan, false) << endl;
}
return true;
}
@ -120,3 +135,10 @@ TEST_F(NewPlannerTest, groupBy) {
bind("SELECT c1 + c3, count(*) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3");
ASSERT_TRUE(run());
}
TEST_F(NewPlannerTest, subquery) {
setDatabase("root", "test");
bind("SELECT count(*) FROM (SELECT c1 + c3 a, c1 + count(*) b FROM t1 where c2 = 'abc' GROUP BY c1, c3) where a > 100 group by b");
ASSERT_TRUE(run());
}

28
source/libs/scalar/inc/filterInt.h
View File

@ -43,7 +43,6 @@ enum {
FLD_TYPE_COLUMN = 1,
FLD_TYPE_VALUE = 2,
FLD_TYPE_MAX = 3,
FLD_DESC_NO_FREE = 4,
FLD_DATA_NO_FREE = 8,
FLD_DATA_IS_HASH = 16,
};
@ -61,11 +60,6 @@ enum {
RANGE_FLG_NULL = 4,
};
enum {
FI_OPTION_NO_REWRITE = 1,
FI_OPTION_TIMESTAMP = 2,
FI_OPTION_NEED_UNIQE = 4,
};
enum {
FI_STATUS_ALL = 1,
@ -107,7 +101,6 @@ typedef struct SFilterRange {
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **);
typedef struct SFilterRangeCompare {
@ -264,12 +257,12 @@ typedef struct SFilterInfo {
} SFilterInfo;
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0)
#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0)
#define SET_AND_OPTR(ctx, o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0)
#define SET_OR_OPTR(ctx,o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0)
#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true)
#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true)))
@ -351,23 +344,10 @@ typedef struct SFilterInfo {
#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL)
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
#if 0
extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
#else
//REMOVE THESE!!!!!!!!!!!!!!!!!!!!
#include "function.h"
#endif
extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
#ifdef __cplusplus
}
#endif

2
source/libs/scalar/inc/sclInt.h
View File

@ -28,6 +28,8 @@ typedef struct SScalarCtx {
SHashObj *pRes; /* element is SScalarParam */
} SScalarCtx;
#define SCL_DATA_TYPE_DUMMY_HASH 9000
#define SCL_DEFAULT_OP_NUM 10
#define sclFatal(...) qFatal(__VA_ARGS__)

1
source/libs/scalar/inc/sclvector.h
View File

@ -24,7 +24,6 @@ extern "C" {
typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order);
_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator);
bool isBinaryStringOp(int32_t op);
#ifdef __cplusplus
}

435
source/libs/scalar/src/filter.c
View File

File diff suppressed because it is too large Load Diff

118
source/libs/scalar/src/scalar.c
View File

@ -7,13 +7,87 @@
#include "sclInt.h"
int32_t scalarGetOperatorParamNum(EOperatorType type) {
if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) {
if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type
|| OP_TYPE_IS_FALSE == type || OP_TYPE_IS_NOT_FALSE == type || OP_TYPE_IS_UNKNOWN == type || OP_TYPE_IS_NOT_UNKNOWN == type) {
return 1;
}
return 2;
}
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
if (NULL == pObj) {
sclError("taosHashInit failed, size:%d", 256);
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type));
int32_t code = 0;
SNodeListNode *nodeList = (SNodeListNode *)pNode;
SListCell *cell = nodeList->pNodeList->pHead;
SScalarParam in = {.num = 1}, out = {.num = 1, .type = type};
int8_t dummy = 0;
int32_t bufLen = 60;
out.data = malloc(bufLen);
int32_t len = 0;
void *buf = NULL;
for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
if (valueNode->node.resType.type != type) {
in.type = valueNode->node.resType.type;
in.bytes = valueNode->node.resType.bytes;
in.data = nodesGetValueFromNode(valueNode);
code = vectorConvertImpl(&in, &out);
if (code) {
sclError("convert from %d to %d failed", in.type, out.type);
SCL_ERR_JRET(code);
}
if (IS_VAR_DATA_TYPE(type)) {
len = varDataLen(out.data);
buf = varDataVal(out.data);
} else {
len = tDataTypes[type].bytes;
buf = out.data;
}
} else {
buf = nodesGetValueFromNode(valueNode);
if (IS_VAR_DATA_TYPE(type)) {
len = varDataLen(buf);
buf = varDataVal(buf);
} else {
len = valueNode->node.resType.bytes;
buf = out.data;
}
}
if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) {
sclError("taosHashPut failed");
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
cell = cell->pNext;
}
tfree(out.data);
*data = pObj;
return TSDB_CODE_SUCCESS;
_return:
tfree(out.data);
taosHashCleanup(pObj);
SCL_RET(code);
}
void sclFreeRes(SHashObj *res) {
SScalarParam *p = NULL;
void *pIter = taosHashIterate(res, NULL);
@ -42,12 +116,22 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
param->num = 1;
param->type = valueNode->node.resType.type;
param->bytes = valueNode->node.resType.bytes;
param->colData = false;
break;
}
case QUERY_NODE_NODE_LIST: {
SNodeListNode *nodeList = (SNodeListNode *)node;
//TODO BUILD HASH
if (nodeList->pNodeList->length <= 0) {
sclError("invalid length in nodeList, length:%d", nodeList->pNodeList->length);
SCL_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SCL_ERR_RET(scalarGenerateSetFromList(&param->data, node, nodeList->dataType.type));
param->num = 1;
param->type = SCL_DATA_TYPE_DUMMY_HASH;
param->colData = false;
break;
}
case QUERY_NODE_COLUMN_REF: {
@ -63,7 +147,14 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
}
SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId);
if (IS_VAR_DATA_TYPE(columnData->info.type)) {
param->data = columnData;
param->colData = true;
} else {
param->data = columnData->pData;
param->colData = false;
}
param->num = ctx->pSrc->info.rows;
param->type = columnData->info.type;
param->bytes = columnData->info.bytes;
@ -248,12 +339,16 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
output->type = node->node.resType.type;
output->bytes = sizeof(bool);
output->num = rowNum;
output->data = calloc(rowNum, sizeof(bool));
if (NULL == output->data) {
sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool)));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
void *data = output->data;
bool value = false;
for (int32_t i = 0; i < rowNum; ++i) {
@ -275,6 +370,8 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
sclParamMoveNext(params, node->pParameterList->length);
}
output->data = data;
return TSDB_CODE_SUCCESS;
_return:
@ -291,6 +388,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
output->type = node->node.resType.type;
output->num = rowNum;
output->bytes = tDataTypes[output->type].bytes;
output->data = calloc(rowNum, tDataTypes[output->type].bytes);
if (NULL == output->data) {
sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes);
@ -303,17 +402,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
SScalarParam* pLeft = &params[0];
SScalarParam* pRight = paramNum > 1 ? &params[1] : NULL;
for (int32_t i = 0; i < rowNum; ++i) {
OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC);
sclParamMoveNext(output, 1);
sclParamMoveNext(pLeft, 1);
if (pRight) {
sclParamMoveNext(pRight, 1);
}
}
return TSDB_CODE_SUCCESS;
_return:
@ -427,7 +517,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) {
EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(*pNode)) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
return DEAL_RES_CONTINUE;
}
@ -509,7 +599,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) {
EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode)) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) {
return DEAL_RES_CONTINUE;
}
@ -525,7 +615,7 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
return sclWalkOperator(pNode, pContext);
}
sclError("invalid node type for calculating constants, type:%d", nodeType(pNode));
sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
SScalarCtx *ctx = (SScalarCtx *)pContext;

3
source/libs/scalar/src/sclfunc.c
View File

@ -268,9 +268,6 @@ static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes,
param->data = pInput;
}
bool isStringOp(int32_t op) {
return op == TSDB_BINARY_OP_CONCAT;
}
#if 0
int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param,

470
source/libs/scalar/src/sclvector.c
View File

@ -21,6 +21,9 @@
#include "querynodes.h"
#include "filterInt.h"
#include "query.h"
#include "sclInt.h"
#include "tep.h"
#include "filter.h"
//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));
@ -93,6 +96,7 @@ double getVectorDoubleValue_FLOAT(void *src, int32_t index) {
double getVectorDoubleValue_DOUBLE(void *src, int32_t index) {
return (double)*((double *)src + index);
}
_getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL;
if(srcType==TSDB_DATA_TYPE_TINYINT) {
@ -218,6 +222,12 @@ void* getVectorValueAddr_FLOAT(void *src, int32_t index) {
void* getVectorValueAddr_DOUBLE(void *src, int32_t index) {
return (void*)((double *)src + index);
}
void* getVectorValueAddr_default(void *src, int32_t index) {
return src;
}
void* getVectorValueAddr_VAR(void *src, int32_t index) {
return colDataGet((SColumnInfoData *)src, index);
}
_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) {
_getValueAddr_fn_t p = NULL;
@ -241,8 +251,12 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) {
p = getVectorValueAddr_FLOAT;
}else if(srcType==TSDB_DATA_TYPE_DOUBLE) {
p = getVectorValueAddr_DOUBLE;
}else if(srcType==TSDB_DATA_TYPE_BINARY) {
p = getVectorValueAddr_VAR;
}else if(srcType==TSDB_DATA_TYPE_NCHAR) {
p = getVectorValueAddr_VAR;
}else {
assert(0);
p = getVectorValueAddr_default;
}
return p;
}
@ -258,31 +272,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
switch (outType) {
case TSDB_DATA_TYPE_BOOL:
if (inType == TSDB_DATA_TYPE_BINARY) {
for (int32_t i = 0; i < pIn->num; ++i) {
GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input));
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += sizeof(bool);
}
} else if (inType == TSDB_DATA_TYPE_NCHAR) {
for (int32_t i = 0; i < pIn->num; ++i) {
GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input));
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
}
} else {
for (int32_t i = 0; i < pIn->num; ++i) {
uint64_t value = 0;
GET_TYPED_DATA(value, uint64_t, inType, input);
SET_TYPED_DATA(output, outType, value);
input += tDataTypes[inType].bytes;
output += tDataTypes[outType].bytes;
}
}
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
@ -291,10 +281,14 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int32_t bufSize = varDataLen(input) + 1;
char *tmp = malloc(bufSize);
if (NULL == tmp) {
sclError("malloc %d failed", bufSize);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
if (varDataLen(input) >= bufSize) {
bufSize = varDataLen(input) + 1;
tmp = realloc(tmp, bufSize);
@ -305,6 +299,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int64_t value = strtoll(tmp, NULL, 10);
SET_TYPED_DATA(output, outType, value);
}
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
@ -315,10 +310,14 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
char *tmp = calloc(1, bufSize);
if (NULL == tmp) {
sclError("calloc %d failed", bufSize);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) {
bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
tmp = realloc(tmp, bufSize);
@ -326,7 +325,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp);
if (len < 0){
qError("castConvert taosUcs4ToMbs error 1");
sclError("castConvert taosUcs4ToMbs error 1");
tfree(tmp);
return TSDB_CODE_QRY_APP_ERROR;
}
@ -334,6 +333,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
tmp[len] = 0;
int64_t value = strtoll(tmp, NULL, 10);
SET_TYPED_DATA(output, outType, value);
}
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
@ -351,6 +351,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
}
}
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
@ -358,10 +359,14 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int32_t bufSize = varDataLen(input) + 1;
char *tmp = malloc(bufSize);
if (NULL == tmp) {
sclError("malloc %d failed", bufSize);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
if (varDataLen(input) >= bufSize) {
bufSize = varDataLen(input) + 1;
tmp = realloc(tmp, bufSize);
@ -371,6 +376,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
tmp[varDataLen(input)] = 0;
uint64_t value = strtoull(tmp, NULL, 10);
SET_TYPED_DATA(output, outType, value);
}
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
@ -381,10 +387,14 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
char *tmp = calloc(1, bufSize);
if (NULL == tmp) {
sclError("calloc %d failed", bufSize);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) {
bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
tmp = realloc(tmp, bufSize);
@ -392,7 +402,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp);
if (len < 0){
qError("castConvert taosUcs4ToMbs error 1");
sclError("castConvert taosUcs4ToMbs error 1");
tfree(tmp);
return TSDB_CODE_QRY_APP_ERROR;
}
@ -400,6 +410,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
tmp[len] = 0;
uint64_t value = strtoull(tmp, NULL, 10);
SET_TYPED_DATA(output, outType, value);
}
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
@ -408,9 +419,13 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
tfree(tmp);
} else {
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
uint64_t value = 0;
GET_TYPED_DATA(value, uint64_t, inType, input);
SET_TYPED_DATA(output, outType, value);
}
input += tDataTypes[inType].bytes;
output += tDataTypes[outType].bytes;
@ -423,10 +438,14 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int32_t bufSize = varDataLen(input) + 1;
char *tmp = malloc(bufSize);
if (NULL == tmp) {
sclError("malloc %d failed", bufSize);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
if (varDataLen(input) >= bufSize) {
bufSize = varDataLen(input) + 1;
tmp = realloc(tmp, bufSize);
@ -437,6 +456,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
double value = strtod(tmp, NULL);
SET_TYPED_DATA(output, outType, value);
}
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
@ -447,10 +467,14 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
char *tmp = calloc(1, bufSize);
if (NULL == tmp) {
sclError("calloc %d failed", bufSize);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) {
bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
tmp = realloc(tmp, bufSize);
@ -458,7 +482,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp);
if (len < 0){
qError("castConvert taosUcs4ToMbs error 1");
sclError("castConvert taosUcs4ToMbs error 1");
tfree(tmp);
return TSDB_CODE_QRY_APP_ERROR;
}
@ -466,6 +490,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
tmp[len] = 0;
double value = strtod(tmp, NULL);
SET_TYPED_DATA(output, outType, value);
}
input += varDataLen(input) + VARSTR_HEADER_SIZE;
output += tDataTypes[outType].bytes;
@ -474,9 +499,13 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
tfree(tmp);
} else {
for (int32_t i = 0; i < pIn->num; ++i) {
if (isNull(input, inType)) {
assignVal(output, getNullValue(outType), 0, outType);
} else {
int64_t value = 0;
GET_TYPED_DATA(value, int64_t, inType, input);
SET_TYPED_DATA(output, outType, value);
}
input += tDataTypes[inType].bytes;
output += tDataTypes[outType].bytes;
@ -484,7 +513,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
}
break;
default:
qError("invalid convert output type:%d", outType);
sclError("invalid convert output type:%d", outType);
return TSDB_CODE_QRY_APP_ERROR;
}
@ -532,6 +561,10 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
return TSDB_CODE_SUCCESS;
}
if (SCL_DATA_TYPE_DUMMY_HASH == pLeft->type || SCL_DATA_TYPE_DUMMY_HASH == pRight->type) {
return TSDB_CODE_SUCCESS;
}
SScalarParam *param1 = NULL, *paramOut1 = NULL;
SScalarParam *param2 = NULL, *paramOut2 = NULL;
int32_t code = 0;
@ -595,6 +628,46 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(double));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(double));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
double *output=(double*)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -628,12 +701,55 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or
SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0));
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(double));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(double));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
double *output=(double*)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -666,11 +782,54 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or
SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0));
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(double));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(double));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
double *output=(double*)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -704,12 +863,55 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_
SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0));
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(double));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(double));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
double *output=(double*)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -750,12 +952,55 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right);
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(double));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(double));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
double * output = (double *)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -822,6 +1067,9 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32
SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right);
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
@ -879,6 +1127,46 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(int64_t));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(int64_t));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
int64_t *output=(int64_t *)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -912,12 +1200,55 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0));
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num};
SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num};
if (IS_VAR_DATA_TYPE(pLeft->type)) {
leftParam.data = calloc(leftParam.num, sizeof(int64_t));
if (NULL == leftParam.data) {
sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t)));
return;
}
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
if (vectorConvertImpl(pLeft, &leftParam)) {
return;
}
pLeft = &leftParam;
}
if (IS_VAR_DATA_TYPE(pRight->type)) {
rightParam.data = calloc(rightParam.num, sizeof(int64_t));
if (NULL == rightParam.data) {
sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t)));
tfree(leftParam.data);
return;
}
if (pRight->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data;
pRight->data = colInfo->pData;
}
if (vectorConvertImpl(pRight, &rightParam)) {
tfree(leftParam.data);
tfree(rightParam.data);
return;
}
pRight = &rightParam;
}
int64_t *output=(int64_t *)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
@ -951,6 +1282,9 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _
SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0));
}
}
tfree(leftParam.data);
tfree(rightParam.data);
}
@ -961,8 +1295,21 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int
bool res = false;
bool *output=(bool *)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
_getValueAddr_fn_t getVectorValueAddrFnLeft = NULL;
_getValueAddr_fn_t getVectorValueAddrFnRight = NULL;
if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) {
getVectorValueAddrFnLeft = getVectorValueAddr_default;
} else {
getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
}
if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) {
getVectorValueAddrFnRight = getVectorValueAddr_default;
} else {
getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
}
if (pLeft->num == pRight->num) {
for (; i < pRight->num && i >= 0; i += step, output += 1) {
@ -1030,55 +1377,55 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
param2 = pRight;
}
vectorCompareImpl(param1, param2, out, _ord, TSDB_RELATION_GREATER);
vectorCompareImpl(param1, param2, out, _ord, optr);
}
void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN);
}
void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL);
}
void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN);
}
void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL);
}
void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL);
}
void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL);
}
void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN);
}
void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN);
}
void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE);
}
void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE);
}
void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH);
}
void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH);
vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH);
}
void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
@ -1087,7 +1434,13 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
bool res = false;
bool *output=(bool *)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnLeft = NULL;
if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) {
getVectorValueAddrFnLeft = getVectorValueAddr_default;
} else {
getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
}
for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) {
@ -1107,7 +1460,13 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
bool res = false;
bool *output = (bool *)out;
_getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
_getValueAddr_fn_t getVectorValueAddrFnLeft = NULL;
if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) {
getVectorValueAddrFnLeft = getVectorValueAddr_default;
} else {
getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
}
for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) {
@ -1121,6 +1480,17 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t
}
}
void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL};
if (pLeft->colData) {
SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data;
pLeft->data = colInfo->pData;
}
vectorConvertImpl(pLeft, &output);
}
_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
switch (binFunctionId) {
@ -1166,12 +1536,12 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return vectorBitAnd;
case OP_TYPE_BIT_OR:
return vectorBitOr;
case OP_TYPE_IS_TRUE:
return vectorIsTrue;
default:
assert(0);
return NULL;
}
}
bool isBinaryStringOp(int32_t op) {
return op == TSDB_BINARY_OP_CONCAT;
}

20
source/libs/scalar/test/CMakeLists.txt
View File

@ -1,18 +1,4 @@
enable_testing()
MESSAGE(STATUS "build scalar unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
scalarTest
PUBLIC os util common gtest qcom function nodes
)
TARGET_INCLUDE_DIRECTORIES(
scalarTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc"
)
add_subdirectory(filter)
add_subdirectory(scalar)

18
source/libs/scalar/test/filter/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build filter unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(filterTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
filterTest
PUBLIC os util common gtest qcom function nodes scalar
)
TARGET_INCLUDE_DIRECTORIES(
filterTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc"
)

581
source/libs/scalar/test/filter/filterTests.cpp Normal file
View File

@ -0,0 +1,581 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "stub.h"
#include "addr_any.h"
#include "scalar.h"
#include "nodes.h"
#include "tlog.h"
#include "filter.h"
namespace {
int64_t flttLeftV = 21, flttRightV = 10;
double flttLeftVd = 21.0, flttRightVd = 10.0;
void flttInitLogFile() {
const char *defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
tsAsyncLog = 0;
qDebugFlag = 159;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_VALUE);
SValueNode *vnode = (SValueNode *)node;
vnode->node.resType.type = dataType;
if (IS_VAR_DATA_TYPE(dataType)) {
vnode->datum.p = (char *)malloc(varDataTLen(value));
varDataCopy(vnode->datum.p, value);
vnode->node.resType.bytes = varDataLen(value);
} else {
vnode->node.resType.bytes = tDataTypes[dataType].bytes;
assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType);
}
*pNode = (SNode *)vnode;
}
void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF);
SColumnRefNode *rnode = (SColumnRefNode *)node;
rnode->dataType.type = dataType;
rnode->dataType.bytes = dataBytes;
rnode->tupleId = 0;
if (NULL == block) {
rnode->slotId = 2;
rnode->columnId = 55;
*pNode = (SNode *)rnode;
return;
}
if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 3;
res->info.rows = rowNum;
res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = {{0}};
idata.info.type = TSDB_DATA_TYPE_NULL;
idata.info.bytes = 10;
idata.info.colId = 0;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)calloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
}
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 55;
idata.pData = (char *)value;
if (IS_VAR_DATA_TYPE(dataType)) {
idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t));
for (int32_t i = 0; i < rowNum; ++i) {
idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i;
}
}
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = 2;
rnode->columnId = 55;
*block = res;
} else {
SSDataBlock *res = *block;
int32_t idx = taosArrayGetSize(res->pDataBlock);
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 55 + idx;
idata.pData = (char *)value;
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = idx;
rnode->columnId = 55 + idx;
}
*pNode = (SNode *)rnode;
}
void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) {
SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR);
SOperatorNode *onode = (SOperatorNode *)node;
onode->node.resType.type = resType;
onode->node.resType.bytes = tDataTypes[resType].bytes;
onode->opType = opType;
onode->pLeft = pLeft;
onode->pRight = pRight;
*pNode = (SNode *)onode;
}
void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) {
SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
SLogicConditionNode *onode = (SLogicConditionNode *)node;
onode->condType = opType;
onode->node.resType.type = TSDB_DATA_TYPE_BOOL;
onode->node.resType.bytes = sizeof(bool);
onode->pParameterList = nodesMakeList();
for (int32_t i = 0; i < nodeNum; ++i) {
nodesListAppend(onode->pParameterList, nodeList[i]);
}
*pNode = (SNode *)onode;
}
void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) {
SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *lnode = (SNodeListNode *)node;
lnode->dataType.type = resType;
lnode->pNodeList = list;
*pNode = (SNode *)lnode;
}
}
TEST(timerangeTest, greater_and_lower) {
flttInitLogFile();
SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL;
bool eRes[5] = {false, false, true, true, true};
SScalarParam res = {0};
int64_t tsmall = 222, tbig = 333;
flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
SNode *list[2] = {0};
list[0] = opNode1;
list[1] = opNode2;
flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP);
ASSERT_EQ(code, 0);
STimeWindow win = {0};
code = filterGetTimeRange(filter, &win);
ASSERT_EQ(code, 0);
ASSERT_EQ(win.skey, tsmall);
ASSERT_EQ(win.ekey, tbig);
}
#if 0
TEST(columnTest, smallint_column_greater_double_value) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int16_t leftv[5]= {1, 2, 3, 4, 5};
double rightv= 2.5;
bool eRes[5] = {false, false, true, true, true};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv);
flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
TEST(columnTest, int_column_in_double_list) {
SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL;
int32_t leftv[5] = {1, 2, 3, 4, 5};
double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3;
bool eRes[5] = {true, true, true, false, false};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv);
SNodeList* list = nodesMakeList();
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1);
nodesListAppend(list, pRight);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2);
nodesListAppend(list, pRight);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3);
nodesListAppend(list, pRight);
flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT);
flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
TEST(columnTest, binary_column_in_binary_list) {
SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL;
bool eRes[5] = {true, true, false, false, false};
SSDataBlock *src = NULL;
SScalarParam res = {0};
char leftv[5][5]= {0};
char rightv[3][5]= {0};
for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = 'a' + i;
leftv[i][3] = 'b' + i;
leftv[i][4] = '0' + i;
varDataSetLen(leftv[i], 3);
}
for (int32_t i = 0; i < 2; ++i) {
rightv[i][2] = 'a' + i;
rightv[i][3] = 'b' + i;
rightv[i][4] = '0' + i;
varDataSetLen(rightv[i], 3);
}
for (int32_t i = 2; i < 3; ++i) {
rightv[i][2] = 'a' + i;
rightv[i][3] = 'a' + i;
rightv[i][4] = 'a' + i;
varDataSetLen(rightv[i], 3);
}
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
SNodeList* list = nodesMakeList();
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]);
nodesListAppend(list, pRight);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]);
nodesListAppend(list, pRight);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]);
nodesListAppend(list, pRight);
flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY);
flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
TEST(columnTest, binary_column_like_binary) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
char rightv[64] = {0};
char leftv[5][5]= {0};
SSDataBlock *src = NULL;
SScalarParam res = {0};
bool eRes[5] = {true, false, true, false, true};
for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = 'a';
leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3);
}
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
sprintf(&rightv[2], "%s", "__0");
varDataSetLen(rightv, strlen(&rightv[2]));
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv);
flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
TEST(columnTest, binary_column_is_null) {
SNode *pLeft = NULL, *opNode = NULL;
char leftv[5][5]= {0};
SSDataBlock *src = NULL;
SScalarParam res = {0};
bool eRes[5] = {false, false, false, false, true};
for (int32_t i = 0; i < 4; ++i) {
leftv[i][2] = '0' + i % 2;
leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3);
}
setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY);
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
TEST(columnTest, binary_column_is_not_null) {
SNode *pLeft = NULL, *opNode = NULL;
char leftv[5][5]= {0};
SSDataBlock *src = NULL;
SScalarParam res = {0};
bool eRes[5] = {true, true, true, true, false};
for (int32_t i = 0; i < 4; ++i) {
leftv[i][2] = '0' + i % 2;
leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3);
}
setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY);
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
TEST(logicTest, and_or_and) {
}
TEST(logicTest, or_and_or) {
}
TEST(opTest, smallint_column_greater_int_column) {
}
TEST(opTest, smallint_value_add_int_column) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int32_t leftv = 1;
int16_t rightv[5]= {0, -5, -4, 23, 100};
double eRes[5] = {1.0, -4, -3, 24, 101};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]);
flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv);
flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((double *)res.data + i), eRes[i]);
}
}
TEST(opTest, bigint_column_multi_binary_column) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int64_t leftv[5]= {1, 2, 3, 4, 5};
char rightv[5][5]= {0};
for (int32_t i = 0; i < 5; ++i) {
rightv[i][2] = rightv[i][3] = '0';
rightv[i][4] = '0' + i;
varDataSetLen(rightv[i], 3);
}
double eRes[5] = {0, 2, 6, 12, 20};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv);
flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((double *)res.data + i), eRes[i]);
}
}
TEST(opTest, smallint_column_and_binary_column) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int16_t leftv[5]= {1, 2, 3, 4, 5};
char rightv[5][5]= {0};
for (int32_t i = 0; i < 5; ++i) {
rightv[i][2] = rightv[i][3] = '0';
rightv[i][4] = '0' + i;
varDataSetLen(rightv[i], 3);
}
int64_t eRes[5] = {0, 0, 2, 0, 4};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv);
flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]);
}
}
TEST(opTest, smallint_column_or_float_column) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int16_t leftv[5]= {1, 2, 3, 4, 5};
float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0};
int64_t eRes[5] = {3, 3, 7, 5, 7};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv);
flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]);
}
}
TEST(opTest, smallint_column_or_double_value) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int16_t leftv[5]= {1, 2, 3, 4, 5};
double rightv= 10.2;
int64_t eRes[5] = {11, 10, 11, 14, 15};
SSDataBlock *src = NULL;
SScalarParam res = {0};
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv);
flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]);
}
}
TEST(opTest, binary_column_is_true) {
SNode *pLeft = NULL, *opNode = NULL;
char leftv[5][5]= {0};
SSDataBlock *src = NULL;
SScalarParam res = {0};
bool eRes[5] = {false, true, false, true, false};
for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = '0' + i % 2;
leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3);
}
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL);
int32_t code = scalarCalculate(opNode, src, &res);
ASSERT_EQ(code, 0);
ASSERT_EQ(res.num, rowNum);
ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((bool *)res.data + i), eRes[i]);
}
}
#endif
int main(int argc, char** argv) {
srand(time(NULL));
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#pragma GCC diagnostic pop

18
source/libs/scalar/test/scalar/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build scalar unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
scalarTest
PUBLIC os util common gtest qcom function nodes scalar
)
TARGET_INCLUDE_DIRECTORIES(
scalarTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc"
)

1183
source/libs/scalar/test/scalar/scalarTests.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

55
source/libs/scalar/test/scalarTests.cpp
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "stub.h"
#include "addr_any.h"
#include "scalar.h"
namespace {
}
TEST(scalarTest, func) {
}
int main(int argc, char** argv) {
srand(time(NULL));
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#pragma GCC diagnostic pop

28
source/util/src/compare.c
View File

@ -466,7 +466,7 @@ int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) {
__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
__compar_fn_t comparFn = NULL;
if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -489,7 +489,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
}
}
if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -522,17 +522,17 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break;
case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break;
case TSDB_DATA_TYPE_BINARY: {
if (optr == TSDB_RELATION_MATCH) {
if (optr == OP_TYPE_MATCH) {
comparFn = compareStrRegexCompMatch;
} else if (optr == TSDB_RELATION_NMATCH) {
} else if (optr == OP_TYPE_NMATCH) {
comparFn = compareStrRegexCompNMatch;
} else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
} else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */
comparFn = compareStrPatternMatch;
} else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */
} else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */
comparFn = compareStrPatternNotMatch;
} else if (optr == TSDB_RELATION_IN) {
} else if (optr == OP_TYPE_IN) {
comparFn = compareChkInString;
} else if (optr == TSDB_RELATION_NOT_IN) {
} else if (optr == OP_TYPE_NOT_IN) {
comparFn = compareChkNotInString;
} else { /* normal relational comparFn */
comparFn = compareLenPrefixedStr;
@ -542,17 +542,17 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
}
case TSDB_DATA_TYPE_NCHAR: {
if (optr == TSDB_RELATION_MATCH) {
if (optr == OP_TYPE_MATCH) {
comparFn = compareStrRegexCompMatch;
} else if (optr == TSDB_RELATION_NMATCH) {
} else if (optr == OP_TYPE_NMATCH) {
comparFn = compareStrRegexCompNMatch;
} else if (optr == TSDB_RELATION_LIKE) {
} else if (optr == OP_TYPE_LIKE) {
comparFn = compareWStrPatternMatch;
} else if (optr == TSDB_RELATION_NOT_LIKE) {
} else if (optr == OP_TYPE_NOT_LIKE) {
comparFn = compareWStrPatternNotMatch;
} else if (optr == TSDB_RELATION_IN) {
} else if (optr == OP_TYPE_IN) {
comparFn = compareChkInString;
} else if (optr == TSDB_RELATION_NOT_IN) {
} else if (optr == OP_TYPE_NOT_IN) {
comparFn = compareChkNotInString;
} else {
comparFn = compareLenPrefixedWStr;