xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into fix/valgrind

This commit is contained in:
Shengliang Guan 2022-05-21 18:37:32 +08:00
parent f4c3141821 591d64aeda
commit 9423000143
17 changed files with 1584 additions and 1455 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmake/cmake.define
View File

@ -71,8 +71,8 @@ ELSE ()
ENDIF () ENDIF ()
IF (${SANITIZER} MATCHES "true") IF (${SANITIZER} MATCHES "true")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
MESSAGE(STATUS "Will compile with Address Sanitizer!") MESSAGE(STATUS "Will compile with Address Sanitizer!")
ELSE () ELSE ()
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")

2
include/common/tmsg.h
View File

@ -2524,7 +2524,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
buf = taosDecodeFixedI32(buf, &pRsp->blockNum); buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
if (pRsp->blockNum != 0) { if (pRsp->blockNum != 0) {

6
include/libs/qcom/query.h
View File

@ -231,19 +231,19 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define qDebug(...) \ #define qDebug(...) \
do { \ do { \
if (qDebugFlag & DEBUG_DEBUG) { \ if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qTrace(...) \ #define qTrace(...) \
do { \ do { \
if (qDebugFlag & DEBUG_TRACE) { \ if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qDebugL(...) \ #define qDebugL(...) \
do { \ do { \
if (qDebugFlag & DEBUG_DEBUG) { \ if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)

568
source/libs/catalog/src/catalog.c
View File

File diff suppressed because it is too large Load Diff

5
source/libs/executor/src/scanoperator.c
View File

@ -972,9 +972,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR
} }
pInfo->primaryTsIndex = 0; // TODO(liuyao) get it from physical plan pInfo->primaryTsIndex = 0; // TODO(liuyao) get it from physical plan
if (pSTInfo->interval.interval > 0) {
pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, 10000); // TODO(liuyao) get watermark from physical plan pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, 10000); // TODO(liuyao) get watermark from physical plan
if (pInfo->pUpdateInfo == NULL) { } else {
goto _error; pInfo->pUpdateInfo = NULL;
} }
pInfo->readHandle = *pHandle; pInfo->readHandle = *pHandle;

4
source/libs/parser/inc/sql.y
View File

@ -241,7 +241,7 @@ alter_table_clause(A) ::=
alter_table_clause(A) ::= alter_table_clause(A) ::=
full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); } full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); }
alter_table_clause(A) ::= alter_table_clause(A) ::=
full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, releaseRawExprNode(pCxt, D)); } full_table_name(B) SET TAG column_name(C) NK_EQ signed_literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); }
%type multi_create_clause { SNodeList* } %type multi_create_clause { SNodeList* }
%destructor multi_create_clause { nodesDestroyList($$); } %destructor multi_create_clause { nodesDestroyList($$); }
@ -448,7 +448,7 @@ agg_func_opt(A) ::= AGGREGATE.
%type bufsize_opt { int32_t } %type bufsize_opt { int32_t }
%destructor bufsize_opt { } %destructor bufsize_opt { }
bufsize_opt(A) ::= . { A = 0; } bufsize_opt(A) ::= . { A = 0; }
bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = strtol(B.z, NULL, 10); } bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = taosStr2Int32(B.z, NULL, 10); }
/************************************************ create/drop stream **************************************************/ /************************************************ create/drop stream **************************************************/
cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A)

146
source/libs/parser/src/parTranslater.c
View File

@ -694,8 +694,22 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
return translateValueImpl(pCxt, pVal, pVal->node.resType); return translateValueImpl(pCxt, pVal, pVal->node.resType);
} }
static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { static bool isMultiResFunc(SNode* pNode) {
if (nodesIsUnaryOp(pOp)) { if (NULL == pNode) {
return false;
}
if (QUERY_NODE_FUNCTION != nodeType(pNode) || !fmIsMultiResFunc(((SFunctionNode*)pNode)->funcId)) {
return false;
}
SNodeList* pParameterList = ((SFunctionNode*)pNode)->pParameterList;
if (LIST_LENGTH(pParameterList) > 1) {
return true;
}
SNode* pParam = nodesListGetNode(pParameterList, 0);
return (QUERY_NODE_COLUMN == nodeType(pParam) ? 0 == strcmp(((SColumnNode*)pParam)->colName, "*") : false);
}
static EDealRes translateUnaryOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
if (OP_TYPE_MINUS == pOp->opType) { if (OP_TYPE_MINUS == pOp->opType) {
if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName);
@ -707,10 +721,11 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
if (nodesIsArithmeticOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
TSDB_DATA_TYPE_BLOB == rdt.type) { TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
@ -731,7 +746,12 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
} }
} else if (nodesIsComparisonOp(pOp)) { return DEAL_RES_CONTINUE;
}
static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
} }
@ -748,12 +768,36 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
} }
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} else if (nodesIsJsonOp(pOp)) { return DEAL_RES_CONTINUE;
}
static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
} }
pOp->node.resType.type = TSDB_DATA_TYPE_JSON; pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
return DEAL_RES_CONTINUE;
}
static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
if (isMultiResFunc(pOp->pLeft)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName);
}
if (isMultiResFunc(pOp->pRight)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
if (nodesIsUnaryOp(pOp)) {
return translateUnaryOperator(pCxt, pOp);
} else if (nodesIsArithmeticOp(pOp)) {
return translateArithmeticOperator(pCxt, pOp);
} else if (nodesIsComparisonOp(pOp)) {
return translateComparisonOperator(pCxt, pOp);
} else if (nodesIsJsonOp(pOp)) {
return translateJsonOperator(pCxt, pOp);
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -808,6 +852,13 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) {
} }
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
SNode* pParam = NULL;
FOREACH(pParam, pFunc->pParameterList) {
if (isMultiResFunc(pParam)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pParam)->aliasName);
}
}
SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog,
.pRpc = pCxt->pParseCxt->pTransporter, .pRpc = pCxt->pParseCxt->pTransporter,
.pMgmtEps = &pCxt->pParseCxt->mgmtEpSet, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet,
@ -926,9 +977,10 @@ typedef struct SCheckExprForGroupByCxt {
STranslateContext* pTranslateCxt; STranslateContext* pTranslateCxt;
int32_t selectFuncNum; int32_t selectFuncNum;
bool hasSelectValFunc; bool hasSelectValFunc;
bool hasOtherAggFunc;
} SCheckExprForGroupByCxt; } SCheckExprForGroupByCxt;
static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSelectValFunc, SNode** pNode) { static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) {
SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) { if (NULL == pFunc) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
@ -942,9 +994,6 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSel
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) { if (TSDB_CODE_SUCCESS == pCxt->errCode) {
*pNode = (SNode*)pFunc; *pNode = (SNode*)pFunc;
if (NULL != pHasSelectValFunc) {
*pHasSelectValFunc = true;
}
} else { } else {
nodesDestroyNode(pFunc); nodesDestroyNode(pFunc);
} }
@ -956,8 +1005,12 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
pCxt->selectFuncNum += isSelectFunc(*pNode) ? 1 : 0; if (isSelectFunc(*pNode)) {
if (pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) { ++(pCxt->selectFuncNum);
} else if (isAggFunc(*pNode)) {
pCxt->hasOtherAggFunc = true;
}
if ((pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) || (pCxt->hasOtherAggFunc && pCxt->hasSelectValFunc)) {
return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt));
} }
if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) {
@ -970,10 +1023,11 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
} }
} }
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
if (pCxt->selectFuncNum > 1) { if (pCxt->selectFuncNum > 1 || pCxt->hasOtherAggFunc) {
return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt));
} else { } else {
return rewriteColToSelectValFunc(pCxt->pTranslateCxt, &pCxt->hasSelectValFunc, pNode); pCxt->hasSelectValFunc = true;
return rewriteColToSelectValFunc(pCxt->pTranslateCxt, pNode);
} }
} }
if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) { if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) {
@ -983,7 +1037,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
} }
static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) { static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) {
SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; SCheckExprForGroupByCxt cxt = {
.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false, .hasOtherAggFunc = false};
nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt); nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt);
if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) {
return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt));
@ -995,7 +1050,8 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList
if (NULL == getGroupByList(pCxt)) { if (NULL == getGroupByList(pCxt)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; SCheckExprForGroupByCxt cxt = {
.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false, .hasOtherAggFunc = false};
nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt); nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt);
if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) {
return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt));
@ -1008,7 +1064,7 @@ static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) {
return DEAL_RES_IGNORE_CHILD; return DEAL_RES_IGNORE_CHILD;
} }
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); return rewriteColToSelectValFunc((STranslateContext*)pContext, pNode);
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -1027,11 +1083,16 @@ typedef struct CheckAggColCoexistCxt {
bool existCol; bool existCol;
bool existNonstdFunc; bool existNonstdFunc;
int32_t selectFuncNum; int32_t selectFuncNum;
bool existOtherAggFunc;
} CheckAggColCoexistCxt; } CheckAggColCoexistCxt;
static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) {
CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext;
pCxt->selectFuncNum += isSelectFunc(pNode) ? 1 : 0; if (isSelectFunc(pNode)) {
++(pCxt->selectFuncNum);
} else if (isAggFunc(pNode)) {
pCxt->existOtherAggFunc = true;
}
if (isAggFunc(pNode)) { if (isAggFunc(pNode)) {
pCxt->existAggFunc = true; pCxt->existAggFunc = true;
return DEAL_RES_IGNORE_CHILD; return DEAL_RES_IGNORE_CHILD;
@ -1050,13 +1111,17 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
if (NULL != pSelect->pGroupByList) { if (NULL != pSelect->pGroupByList) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
CheckAggColCoexistCxt cxt = { CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt,
.pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false, .existNonstdFunc = false}; .existAggFunc = false,
.existCol = false,
.existNonstdFunc = false,
.selectFuncNum = 0,
.existOtherAggFunc = false};
nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt);
if (!pSelect->isDistinct) { if (!pSelect->isDistinct) {
nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt);
} }
if (1 == cxt.selectFuncNum) { if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) {
return rewriteColsToSelectValFunc(pCxt, pSelect); return rewriteColsToSelectValFunc(pCxt, pSelect);
} }
if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) {
@ -1230,18 +1295,6 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool isMultiResFunc(SNode* pNode) {
if (QUERY_NODE_FUNCTION != nodeType(pNode) || !fmIsMultiResFunc(((SFunctionNode*)pNode)->funcId)) {
return false;
}
SNodeList* pParameterList = ((SFunctionNode*)pNode)->pParameterList;
if (LIST_LENGTH(pParameterList) > 1) {
return true;
}
SNode* pParam = nodesListGetNode(pParameterList, 0);
return (QUERY_NODE_COLUMN == nodeType(pParam) ? 0 == strcmp(((SColumnNode*)pParam)->colName, "*") : false);
}
static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) {
SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) { if (NULL == pFunc) {
@ -1872,7 +1925,7 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) {
} }
static bool dataTypeEqual(const SDataType* l, const SDataType* r) { static bool dataTypeEqual(const SDataType* l, const SDataType* r) {
return (l->type == r->type && l->bytes == l->bytes && l->precision == r->precision && l->scale == l->scale); return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale);
} }
static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) { static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) {
@ -2726,8 +2779,11 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm
SName tableName; SName tableName;
tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name);
int32_t code = collectUseTable(&tableName, pCxt->pTables);
return buildRollupAst(pCxt, pStmt, pReq); if (TSDB_CODE_SUCCESS == code) {
code = buildRollupAst(pCxt, pStmt, pReq);
}
return code;
} }
static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
@ -4032,13 +4088,18 @@ static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* p
return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal);
} }
static int32_t colDataBytesToValueDataBytes(uint8_t type, int32_t bytes) {
if (TSDB_DATA_TYPE_VARCHAR == type || TSDB_DATA_TYPE_BINARY == type || TSDB_DATA_TYPE_VARBINARY == type) {
return bytes - VARSTR_HEADER_SIZE;
} else if (TSDB_DATA_TYPE_NCHAR == type) {
return (bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
}
return bytes;
}
static SDataType schemaToDataType(SSchema* pSchema) { static SDataType schemaToDataType(SSchema* pSchema) {
SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0}; SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0};
if (TSDB_DATA_TYPE_VARCHAR == dt.type || TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type) { dt.bytes = colDataBytesToValueDataBytes(pSchema->type, pSchema->bytes);
dt.bytes -= VARSTR_HEADER_SIZE;
} else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
dt.bytes = (dt.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
}
return dt; return dt;
} }
@ -4440,7 +4501,8 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt
SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
if (NULL == pSchema) { if (NULL == pSchema) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
} else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->bytes >= pReq->colModBytes) { } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
pSchema->bytes >= pReq->colModBytes) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);
} }

1328
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

4
source/libs/parser/test/parSelectTest.cpp
View File

@ -239,6 +239,10 @@ TEST_F(ParserSelectTest, semanticError) {
// TSDB_CODE_PAR_WRONG_VALUE_TYPE // TSDB_CODE_PAR_WRONG_VALUE_TYPE
run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE);
run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE);
run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE);
// TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION
run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION,
PARSER_STAGE_TRANSLATE); PARSER_STAGE_TRANSLATE);

18
source/libs/planner/src/planSpliter.c
View File

@ -248,6 +248,7 @@ static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode)
pSubplan->id.groupId = pCxt->groupId; pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
pSubplan->pNode = pNode; pSubplan->pNode = pNode;
pNode->pParent = NULL;
return pSubplan; return pSubplan;
} }
@ -408,17 +409,30 @@ static const SSplitRule splitRuleSet[] = {{.pName = "SuperTableScan", .splitFunc
static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
char* pStr = NULL;
nodesNodeToString(pSubplan, false, &pStr, NULL);
qDebugL("apply %s rule: %s", pRuleName, pStr);
taosMemoryFree(pStr);
}
static int32_t applySplitRule(SLogicSubplan* pSubplan) { static int32_t applySplitRule(SLogicSubplan* pSubplan) {
SSplitContext cxt = {.queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false}; SSplitContext cxt = {.queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false};
bool split = false;
do { do {
cxt.split = false; split = false;
for (int32_t i = 0; i < splitRuleNum; ++i) { for (int32_t i = 0; i < splitRuleNum; ++i) {
cxt.split = false;
int32_t code = splitRuleSet[i].splitFunc(&cxt, pSubplan); int32_t code = splitRuleSet[i].splitFunc(&cxt, pSubplan);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
if (cxt.split) {
split = true;
dumpLogicSubplan(splitRuleSet[i].pName, pSubplan);
} }
} while (cxt.split); }
} while (split);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

19
source/libs/planner/test/planSetOpTest.cpp
View File

@ -23,7 +23,12 @@ class PlanSetOpTest : public PlannerTestBase {};
TEST_F(PlanSetOpTest, unionAll) { TEST_F(PlanSetOpTest, unionAll) {
useDb("root", "test"); useDb("root", "test");
// sql 1: single UNION ALL operator
run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20"); run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20");
// sql 2: multi UNION ALL operator
run("SELECT c1, c2 FROM t1 WHERE c1 > 10 "
"UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 "
"UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 30");
} }
TEST_F(PlanSetOpTest, unionAllSubquery) { TEST_F(PlanSetOpTest, unionAllSubquery) {
@ -44,7 +49,12 @@ TEST_F(PlanSetOpTest, unionAllWithSubquery) {
TEST_F(PlanSetOpTest, union) { TEST_F(PlanSetOpTest, union) {
useDb("root", "test"); useDb("root", "test");
// single UNION operator
run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION SELECT c1, c2 FROM t1 WHERE c1 > 20"); run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION SELECT c1, c2 FROM t1 WHERE c1 > 20");
// multi UNION operator
run("SELECT c1, c2 FROM t1 WHERE c1 > 10 "
"UNION SELECT c1, c2 FROM t1 WHERE c1 > 20 "
"UNION SELECT c1, c2 FROM t1 WHERE c1 > 30");
} }
TEST_F(PlanSetOpTest, unionContainJoin) { TEST_F(PlanSetOpTest, unionContainJoin) {
@ -62,3 +72,12 @@ TEST_F(PlanSetOpTest, unionSubquery) {
run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION SELECT c1, c2 FROM t1)"); run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION SELECT c1, c2 FROM t1)");
} }
TEST_F(PlanSetOpTest, bug001) {
useDb("root", "test");
run("SELECT c2 FROM t1 WHERE c1 IS NOT NULL GROUP BY c2 "
"UNION "
"SELECT 'abcdefghijklmnopqrstuvwxyz' FROM t1 "
"WHERE 'abcdefghijklmnopqrstuvwxyz' IS NOT NULL GROUP BY 'abcdefghijklmnopqrstuvwxyz'");
}

2
source/libs/planner/test/planSubqueryTest.cpp
View File

@ -23,9 +23,7 @@ class PlanSubqeuryTest : public PlannerTestBase {};
TEST_F(PlanSubqeuryTest, basic) { TEST_F(PlanSubqeuryTest, basic) {
useDb("root", "test"); useDb("root", "test");
if (0 == g_skipSql) {
run("SELECT * FROM (SELECT * FROM t1)"); run("SELECT * FROM (SELECT * FROM t1)");
}
run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); run("SELECT LAST(c1) FROM (SELECT * FROM t1)");
} }

38
source/libs/planner/test/planTestMain.cpp
View File

@ -35,18 +35,19 @@ class PlannerEnv : public testing::Environment {
private: private:
void initLog(const char* path) { void initLog(const char* path) {
dDebugFlag = 143; int32_t logLevel = getLogLevel();
vDebugFlag = 0; dDebugFlag = logLevel;
mDebugFlag = 143; vDebugFlag = logLevel;
cDebugFlag = 0; mDebugFlag = logLevel;
jniDebugFlag = 0; cDebugFlag = logLevel;
tmrDebugFlag = 135; jniDebugFlag = logLevel;
uDebugFlag = 135; tmrDebugFlag = logLevel;
rpcDebugFlag = 143; uDebugFlag = logLevel;
qDebugFlag = 143; rpcDebugFlag = logLevel;
wDebugFlag = 0; qDebugFlag = logLevel;
sDebugFlag = 0; wDebugFlag = logLevel;
tsdbDebugFlag = 0; sDebugFlag = logLevel;
tsdbDebugFlag = logLevel;
tsLogEmbedded = 1; tsLogEmbedded = 1;
tsAsyncLog = 0; tsAsyncLog = 0;
@ -62,15 +63,24 @@ class PlannerEnv : public testing::Environment {
static void parseArg(int argc, char* argv[]) { static void parseArg(int argc, char* argv[]) {
int opt = 0; int opt = 0;
const char* optstring = ""; const char* optstring = "";
// clang-format off
static struct option long_options[] = { static struct option long_options[] = {
{"dump", optional_argument, NULL, 'd'}, {"skipSql", optional_argument, NULL, 's'}, {0, 0, 0, 0}}; {"dump", optional_argument, NULL, 'd'},
{"skipSql", required_argument, NULL, 's'},
{"log", required_argument, NULL, 'l'},
{0, 0, 0, 0}
};
// clang-format on
while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) {
switch (opt) { switch (opt) {
case 'd': case 'd':
setDumpModule(optarg); setDumpModule(optarg);
break; break;
case 's': case 's':
g_skipSql = 1; setSkipSqlNum(optarg);
break;
case 'l':
setLogLevel(optarg);
break; break;
default: default:
break; break;

27
source/libs/planner/test/planTestUtil.cpp
View File

@ -48,6 +48,7 @@ enum DumpModule {
DumpModule g_dumpModule = DUMP_MODULE_NOTHING; DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
int32_t g_skipSql = 0; int32_t g_skipSql = 0;
int32_t g_logLevel = 131;
void setDumpModule(const char* pModule) { void setDumpModule(const char* pModule) {
if (NULL == pModule) { if (NULL == pModule) {
@ -71,14 +72,26 @@ void setDumpModule(const char* pModule) {
} }
} }
void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(optarg); }
void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); }
int32_t getLogLevel() { return g_logLevel; }
class PlannerTestBaseImpl { class PlannerTestBaseImpl {
public: public:
void useDb(const string& acctId, const string& db) { void useDb(const string& acctId, const string& db) {
caseEnv_.acctId_ = acctId; caseEnv_.acctId_ = acctId;
caseEnv_.db_ = db; caseEnv_.db_ = db;
caseEnv_.nsql_ = g_skipSql;
} }
void run(const string& sql) { void run(const string& sql) {
if (caseEnv_.nsql_ > 0) {
--(caseEnv_.nsql_);
return;
}
reset(); reset();
try { try {
SQuery* pQuery = nullptr; SQuery* pQuery = nullptr;
@ -109,6 +122,10 @@ class PlannerTestBaseImpl {
} }
void prepare(const string& sql) { void prepare(const string& sql) {
if (caseEnv_.nsql_ > 0) {
return;
}
reset(); reset();
try { try {
doParseSql(sql, &stmtEnv_.pQuery_, true); doParseSql(sql, &stmtEnv_.pQuery_, true);
@ -119,6 +136,10 @@ class PlannerTestBaseImpl {
} }
void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) {
if (caseEnv_.nsql_ > 0) {
return;
}
try { try {
doBindParams(stmtEnv_.pQuery_, pParams, colIdx); doBindParams(stmtEnv_.pQuery_, pParams, colIdx);
} catch (...) { } catch (...) {
@ -128,6 +149,11 @@ class PlannerTestBaseImpl {
} }
void exec() { void exec() {
if (caseEnv_.nsql_ > 0) {
--(caseEnv_.nsql_);
return;
}
try { try {
doParseBoundSql(stmtEnv_.pQuery_); doParseBoundSql(stmtEnv_.pQuery_);
@ -159,6 +185,7 @@ class PlannerTestBaseImpl {
struct caseEnv { struct caseEnv {
string acctId_; string acctId_;
string db_; string db_;
int32_t nsql_;
}; };
struct stmtEnv { struct stmtEnv {

5
source/libs/planner/test/planTestUtil.h
View File

@ -37,8 +37,9 @@ class PlannerTestBase : public testing::Test {
std::unique_ptr<PlannerTestBaseImpl> impl_; std::unique_ptr<PlannerTestBaseImpl> impl_;
}; };
extern int32_t g_skipSql;
extern void setDumpModule(const char* pModule); extern void setDumpModule(const char* pModule);
extern void setSkipSqlNum(const char* pNum);
extern void setLogLevel(const char* pLogLevel);
extern int32_t getLogLevel();
#endif // PLAN_TEST_UTIL_H #endif // PLAN_TEST_UTIL_H

2
source/libs/sync/src/syncRespMgr.c
View File

@ -76,8 +76,8 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu
void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index));
if (pTmp != NULL) { if (pTmp != NULL) {
memcpy(pStub, pTmp, sizeof(SRespStub)); memcpy(pStub, pTmp, sizeof(SRespStub));
taosThreadMutexUnlock(&(pObj->mutex));
taosHashRemove(pObj->pRespHash, &index, sizeof(index)); taosHashRemove(pObj->pRespHash, &index, sizeof(index));
taosThreadMutexUnlock(&(pObj->mutex));
return 1; // get one object return 1; // get one object
} }
taosThreadMutexUnlock(&(pObj->mutex)); taosThreadMutexUnlock(&(pObj->mutex));

111
source/util/src/thash.c
View File

@ -15,8 +15,8 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "thash.h" #include "thash.h"
#include "taoserror.h"
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "tlog.h" #include "tlog.h"
// the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT
@ -27,13 +27,13 @@
#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) #define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) #define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) #define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
#define FREE_HASH_NODE(_fp, _n) \ #define FREE_HASH_NODE(_fp, _n) \
do { \ do { \
/* if (_fp != NULL) { \ /* if (_fp != NULL) { \
(_fp)(_n); \ (_fp)(_n); \
}*/ \ }*/ \
taosMemoryFreeClear(_n); \ taosMemoryFreeClear(_n); \
@ -56,7 +56,7 @@ typedef struct SHashEntry {
} SHashEntry; } SHashEntry;
struct SHashObj { struct SHashObj {
SHashEntry ** hashList; SHashEntry **hashList;
size_t capacity; // number of slots size_t capacity; // number of slots
int64_t size; // number of elements in hash table int64_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function _hash_fn_t hashFp; // hash function
@ -65,7 +65,7 @@ struct SHashObj {
SRWLatch lock; // read-write spin lock SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type SHashLockTypeE type; // lock type
bool enableUpdate; // enable update bool enableUpdate; // enable update
SArray * pMemBlock; // memory block allocated for SHashEntry SArray *pMemBlock; // memory block allocated for SHashEntry
_hash_before_fn_t callbackFp; // function invoked before return the value to caller _hash_before_fn_t callbackFp; // function invoked before return the value to caller
}; };
@ -103,14 +103,14 @@ static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) {
taosRUnLockLatch(&pHashObj->lock); taosRUnLockLatch(&pHashObj->lock);
} }
static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) { static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) { if (pHashObj->type == HASH_NO_LOCK) {
return; return;
} }
taosWLockLatch(&pe->latch); taosWLockLatch(&pe->latch);
} }
static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) { static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) { if (pHashObj->type == HASH_NO_LOCK) {
return; return;
} }
@ -118,7 +118,7 @@ static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEnt
taosWUnLockLatch(&pe->latch); taosWUnLockLatch(&pe->latch);
} }
static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) { static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) { if (pHashObj->type == HASH_NO_LOCK) {
return; return;
} }
@ -126,7 +126,7 @@ static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry
taosRLockLatch(&pe->latch); taosRLockLatch(&pe->latch);
} }
static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) { static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) { if (pHashObj->type == HASH_NO_LOCK) {
return; return;
} }
@ -142,12 +142,11 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
return i; return i;
} }
static FORCE_INLINE SHashNode * static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen,
doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { uint32_t hashVal) {
SHashNode *pNode = pe->next; SHashNode *pNode = pe->next;
while (pNode) { while (pNode) {
if ((pNode->keyLen == keyLen) && if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
pNode->removed == 0) { pNode->removed == 0) {
assert(pNode->hashVal == hashVal); assert(pNode->hashVal == hashVal);
break; break;
@ -186,18 +185,22 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
* @param pNode the old node with requested key * @param pNode the old node with requested key
* @param pNewNode the new node with requested key * @param pNewNode the new node with requested key
*/ */
static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode,
SHashNode *pNewNode) {
assert(pNode->keyLen == pNewNode->keyLen); assert(pNode->keyLen == pNewNode->keyLen);
atomic_sub_fetch_16(&pNode->refCount, 1); atomic_sub_fetch_16(&pNode->refCount, 1);
if (prev != NULL) { if (prev != NULL) {
prev->next = pNewNode; prev->next = pNewNode;
ASSERT(prev->next != prev);
} else { } else {
pe->next = pNewNode; pe->next = pNewNode;
} }
if (pNode->refCount <= 0) { if (pNode->refCount <= 0) {
pNewNode->next = pNode->next; pNewNode->next = pNode->next;
ASSERT(pNewNode->next != pNewNode);
FREE_HASH_NODE(pHashObj->freeFp, pNode); FREE_HASH_NODE(pHashObj->freeFp, pNode);
} else { } else {
pNewNode->next = pNode; pNewNode->next = pNode;
@ -227,9 +230,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj);
* @param pHashObj * @param pHashObj
* @return * @return
*/ */
static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; }
return taosHashGetSize(pHashObj) == 0;
}
SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
if (fn == NULL) { if (fn == NULL) {
@ -305,7 +306,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
if (pHashObj == NULL) { if (pHashObj == NULL) {
return 0; return 0;
} }
return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); return (int32_t)atomic_load_64((int64_t *)&pHashObj->size);
} }
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) {
@ -340,10 +341,9 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
} }
#endif #endif
SHashNode* prev = NULL; SHashNode *prev = NULL;
while (pNode) { while (pNode) {
if ((pNode->keyLen == keyLen) && if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
(*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
pNode->removed == 0) { pNode->removed == 0) {
assert(pNode->hashVal == hashVal); assert(pNode->hashVal == hashVal);
break; break;
@ -391,27 +391,27 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
} }
} }
static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef); static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef);
void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
void* p = NULL; void *p = NULL;
return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false); return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false);
} }
int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) { int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) {
terrno = 0; terrno = 0;
/*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false);
return terrno; return terrno;
} }
int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) { int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t *size) {
terrno = 0; terrno = 0;
/*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false);
return terrno; return terrno;
} }
void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) { void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
return NULL; return NULL;
} }
@ -457,7 +457,7 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void**
} }
} else if (*size < pNode->dataLen) { } else if (*size < pNode->dataLen) {
*size = pNode->dataLen; *size = pNode->dataLen;
char* tmp = taosMemoryRealloc(*d, *size); char *tmp = taosMemoryRealloc(*d, *size);
if (tmp == NULL) { if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -513,8 +513,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
SHashNode *prevNode = NULL; SHashNode *prevNode = NULL;
while (pNode) { while (pNode) {
if ((pNode->keyLen == keyLen) && if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
pNode->removed == 0) { pNode->removed == 0) {
code = 0; // it is found code = 0; // it is found
@ -525,6 +524,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
pe->next = pNode->next; pe->next = pNode->next;
} else { } else {
prevNode->next = pNode->next; prevNode->next = pNode->next;
ASSERT(prevNode->next != prevNode);
} }
pe->num--; pe->num--;
@ -598,14 +598,14 @@ void taosHashCleanup(SHashObj *pHashObj) {
} }
// for profile only // for profile only
int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) {
return 0; return 0;
} }
int32_t num = 0; int32_t num = 0;
taosHashRLock((SHashObj*) pHashObj); taosHashRLock((SHashObj *)pHashObj);
for (int32_t i = 0; i < pHashObj->size; ++i) { for (int32_t i = 0; i < pHashObj->size; ++i) {
SHashEntry *pEntry = pHashObj->hashList[i]; SHashEntry *pEntry = pHashObj->hashList[i];
@ -616,7 +616,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){
} }
} }
taosHashRUnlock((SHashObj*) pHashObj); taosHashRUnlock((SHashObj *)pHashObj);
return num; return num;
} }
@ -627,22 +627,22 @@ void taosHashTableResize(SHashObj *pHashObj) {
int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
if (newCapacity > HASH_MAX_CAPACITY) { if (newCapacity > HASH_MAX_CAPACITY) {
// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
// pHashObj->capacity, HASH_MAX_CAPACITY); // pHashObj->capacity, HASH_MAX_CAPACITY);
return; return;
} }
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
if (pNewEntryList == NULL) { if (pNewEntryList == NULL) {
// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
return; return;
} }
pHashObj->hashList = pNewEntryList; pHashObj->hashList = pNewEntryList;
size_t inc = newCapacity - pHashObj->capacity; size_t inc = newCapacity - pHashObj->capacity;
void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); void *p = taosMemoryCalloc(inc, sizeof(SHashEntry));
for (int32_t i = 0; i < inc; ++i) { for (int32_t i = 0; i < inc; ++i) {
pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
@ -688,12 +688,13 @@ void taosHashTableResize(SHashObj *pHashObj) {
int64_t et = taosGetTimestampUs(); int64_t et = taosGetTimestampUs();
// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms",
// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); // (int32_t)pHashObj->capacity,
// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
} }
SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize); SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize + 1);
if (pNewNode == NULL) { if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -703,7 +704,7 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
pNewNode->keyLen = (uint32_t)keyLen; pNewNode->keyLen = (uint32_t)keyLen;
pNewNode->hashVal = hashVal; pNewNode->hashVal = hashVal;
pNewNode->dataLen = (uint32_t)dsize; pNewNode->dataLen = (uint32_t)dsize;
pNewNode->refCount= 1; pNewNode->refCount = 1;
pNewNode->removed = 0; pNewNode->removed = 0;
pNewNode->next = NULL; pNewNode->next = NULL;
@ -719,6 +720,7 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) {
pNode->next = pEntry->next; pNode->next = pEntry->next;
pEntry->next = pNode; pEntry->next = pNode;
ASSERT(pNode->next != pNode);
pEntry->num += 1; pEntry->num += 1;
} }
@ -727,11 +729,12 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
return 0; return 0;
} }
return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void *))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) +
sizeof(SHashObj);
} }
void *taosHashGetKey(void *data, size_t* keyLen) { void *taosHashGetKey(void *data, size_t *keyLen) {
SHashNode * node = GET_HASH_PNODE(data); SHashNode *node = GET_HASH_PNODE(data);
if (keyLen != NULL) { if (keyLen != NULL) {
*keyLen = node->keyLen; *keyLen = node->keyLen;
} }
@ -751,8 +754,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
SHashNode *pNode = pe->next; SHashNode *pNode = pe->next;
while (pNode) { while (pNode) {
if (pNode == pOld) if (pNode == pOld) break;
break;
prevNode = pNode; prevNode = pNode;
pNode = pNode->next; pNode = pNode->next;
@ -766,11 +768,16 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
} }
atomic_sub_fetch_16(&pOld->refCount, 1); atomic_sub_fetch_16(&pOld->refCount, 1);
if (pOld->refCount <=0) { if (pOld->refCount <= 0) {
if (prevNode) { if (prevNode) {
prevNode->next = pOld->next; prevNode->next = pOld->next;
ASSERT(prevNode->next != prevNode);
} else { } else {
pe->next = pOld->next; pe->next = pOld->next;
SHashNode* x = pe->next;
if (x != NULL) {
ASSERT(x->next != x);
}
} }
pe->num--; pe->num--;
@ -778,7 +785,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
FREE_HASH_NODE(pHashObj->freeFp, pOld); FREE_HASH_NODE(pHashObj->freeFp, pOld);
} }
} else { } else {
// uError("pNode:%p data:%p is not there!!!", pNode, p); // uError("pNode:%p data:%p is not there!!!", pNode, p);
} }
return pNode; return pNode;
@ -865,9 +872,9 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
taosHashRUnlock(pHashObj); taosHashRUnlock(pHashObj);
} }
//TODO remove it // TODO remove it
void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
void* p = NULL; void *p = NULL;
return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true); return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true);
} }