From 34aab8ea62b76072c38c3ad321959e273b947071 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 19:43:35 +0800 Subject: [PATCH 01/62] enh: downgrade value node type --- source/libs/scalar/inc/sclInt.h | 3 +++ source/libs/scalar/src/scalar.c | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index d423b92da7..1a0e633e4a 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -31,6 +31,7 @@ typedef struct SOperatorValueType { typedef struct SScalarCtx { int32_t code; bool dual; + bool compOp; SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values @@ -45,6 +46,8 @@ typedef struct SScalarCtx { #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) //#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0)) #define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type)) +#define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN) +#define SCL_DOWNGRADE_DATETYPE(_type) ((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT) #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d0c5a76f4b..dc545c79ed 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -219,6 +219,18 @@ void sclFreeParamList(SScalarParam *param, int32_t paramNum) { taosMemoryFree(param); } +void sclDowngradeValueType(SValueNode *valueNode) { + switch (valueNode->node.resType.type) { + case TSDB_DATA_TYPE_BIGINT: { + + } + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + default: + break; + } +} + int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_LEFT_VALUE: { @@ -231,6 +243,10 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t ASSERT(param->columnData == NULL); param->numOfRows = 1; + if (ctx->compOp && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { + sclDowngradeValueType(valueNode); + } + /*int32_t code = */sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { colDataAppendNULL(param->columnData, 0); @@ -437,6 +453,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal } sclSetOperatorValueType(node, ctx); + ctx->compOp = SCL_IS_COMPARISON_OPERATOR(node->opType); SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); if (paramNum > 1) { From c2d2f554f22df2ad6d2a55cb99074243ebfcf7f4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 8 Aug 2022 09:14:50 +0800 Subject: [PATCH 02/62] enh: downgrade value type to speed filter --- source/libs/scalar/inc/sclInt.h | 1 - source/libs/scalar/src/scalar.c | 85 +++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 1a0e633e4a..36d2c5a49c 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -31,7 +31,6 @@ typedef struct SOperatorValueType { typedef struct SScalarCtx { int32_t code; bool dual; - bool compOp; SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index dc545c79ed..f08677084d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -9,6 +9,7 @@ #include "scalar.h" #include "tudf.h" #include "ttime.h" +#include "tcompare.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type @@ -222,10 +223,74 @@ void sclFreeParamList(SScalarParam *param, int32_t paramNum) { void sclDowngradeValueType(SValueNode *valueNode) { switch (valueNode->node.resType.type) { case TSDB_DATA_TYPE_BIGINT: { - + int8_t i8 = valueNode->datum.i; + if (i8 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_TINYINT; + *(int8_t*)&valueNode->typeData = i8; + break; + } + int16_t i16 = valueNode->datum.i; + if (i16 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_SMALLINT; + *(int16_t*)&valueNode->typeData = i16; + break; + } + int32_t i32 = valueNode->datum.i; + if (i32 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_INT; + *(int32_t*)&valueNode->typeData = i32; + break; + } + break; + } + case TSDB_DATA_TYPE_UBIGINT:{ + uint8_t u8 = valueNode->datum.i; + if (u8 == valueNode->datum.i) { + int8_t i8 = valueNode->datum.i; + if (i8 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_TINYINT; + *(int8_t*)&valueNode->typeData = i8; + } else { + valueNode->node.resType.type = TSDB_DATA_TYPE_UTINYINT; + *(uint8_t*)&valueNode->typeData = u8; + } + break; + } + uint16_t u16 = valueNode->datum.i; + if (u16 == valueNode->datum.i) { + int16_t i16 = valueNode->datum.i; + if (i16 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_SMALLINT; + *(int16_t*)&valueNode->typeData = i16; + } else { + valueNode->node.resType.type = TSDB_DATA_TYPE_USMALLINT; + *(uint16_t*)&valueNode->typeData = u16; + } + break; + } + uint32_t u32 = valueNode->datum.i; + if (u32 == valueNode->datum.i) { + int32_t i32 = valueNode->datum.i; + if (i32 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_INT; + *(int32_t*)&valueNode->typeData = i32; + } else { + valueNode->node.resType.type = TSDB_DATA_TYPE_UINT; + *(uint32_t*)&valueNode->typeData = u32; + } + break; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + float f = valueNode->datum.d; + if (FLT_EQUAL(f, valueNode->datum.d)) { + valueNode->node.resType.type = TSDB_DATA_TYPE_FLOAT; + *(float*)&valueNode->typeData = f; + break; + } + break; } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: default: break; } @@ -243,10 +308,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t ASSERT(param->columnData == NULL); param->numOfRows = 1; - if (ctx->compOp && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { - sclDowngradeValueType(valueNode); - } - + /*int32_t code = */sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { colDataAppendNULL(param->columnData, 0); @@ -453,7 +515,6 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal } sclSetOperatorValueType(node, ctx); - ctx->compOp = SCL_IS_COMPARISON_OPERATOR(node->opType); SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); if (paramNum > 1) { @@ -689,6 +750,10 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } } + + if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { + sclDowngradeValueType(valueNode); + } } if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { @@ -706,6 +771,10 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } } + + if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { + sclDowngradeValueType(valueNode); + } } if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) { From 1ee558fb63a54df0c21a5280e62c402d118decae Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 8 Aug 2022 15:20:12 +0800 Subject: [PATCH 03/62] fix(query): interp + fill(linear) not working TD-18220 --- source/libs/executor/inc/executorimpl.h | 25 ++--- source/libs/executor/inc/tfill.h | 8 +- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 102 ++++++++++++++++-- 4 files changed, 116 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f8a7ab330f..262ea9f79a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -732,18 +732,19 @@ typedef struct SStreamSessionAggOperatorInfo { } SStreamSessionAggOperatorInfo; typedef struct STimeSliceOperatorInfo { - SSDataBlock* pRes; - STimeWindow win; - SInterval interval; - int64_t current; - SArray* pPrevRow; // SArray - SArray* pNextRow; // SArray - bool isPrevRowSet; - bool isNextRowSet; - int32_t fillType; // fill type - SColumn tsCol; // primary timestamp column - SExprSupp scalarSup; // scalar calculation - struct SFillColInfo* pFillColInfo; // fill column info + SSDataBlock* pRes; + STimeWindow win; + SInterval interval; + int64_t current; + SArray* pPrevRow; // SArray + SArray* pNextRow; // SArray + SArray* pLinearInfo; // SArray + bool isPrevRowSet; + bool isNextRowSet; + int32_t fillType; // fill type + SColumn tsCol; // primary timestamp column + SExprSupp scalarSup; // scalar calculation + struct SFillColInfo* pFillColInfo; // fill column info } STimeSliceOperatorInfo; typedef struct SStateWindowOperatorInfo { diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index b604794dad..a2beedba36 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -33,11 +33,17 @@ typedef struct SFillColInfo { SVariant fillVal; } SFillColInfo; +typedef struct SFillLinearInfo { + SPoint start; + SPoint end; + bool fillLastPoint; +} SFillLinearInfo; + typedef struct { SSchema col; char* tagVal; } SFillTagColInfo; - + typedef struct SFillInfo { TSKEY start; // start timestamp TSKEY end; // endKey for fill diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index c5d68676d2..bc266cc33e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -669,4 +669,4 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const str } return pFillCol; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index e7d9a8d8b2..afc5849963 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2087,6 +2087,24 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pSliceInfo->isNextRowSet = true; } +static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + // null data should not be kept since it can not be used to perform interpolation + if (!colDataIsNull_s(pColInfoData, i)) { + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + memcpy(pkey->pData, val, pkey->bytes); + } + } + + pSliceInfo->isNextRowSet = true; +} + static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; @@ -2246,6 +2264,52 @@ static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB return TSDB_CODE_SUCCESS; } +static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { + if (pInfo->pLinearInfo != NULL) { + return TSDB_CODE_SUCCESS; + } + + pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo)); + if (pInfo->pNextRow == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SFillLinearInfo pLinearInfo = {0}; + pLinearInfo.start.key = INT64_MIN; + pLinearInfo.end.key = INT64_MAX; + pLinearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); + pLinearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); + pLinearInfo.fillLastPoint = false; + taosArrayPush(pInfo->pLinearInfo, &pLinearInfo); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { + int32_t code; + code = initPrevRowsKeeper(pInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + + code = initNextRowsKeeper(pInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + + code = initFillLinearInfo(pInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2278,13 +2342,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } - int32_t code; - code = initPrevRowsKeeper(pSliceInfo, pBlock); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - code = initNextRowsKeeper(pSliceInfo, pBlock); + int32_t code = initKeeperInfo(pSliceInfo, pBlock); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -2312,6 +2370,33 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + // for linear interpolation, always fill value between this and next points; + // if its the first point in data block, also fill values between previous(if there's any) and this point; + // if its the last point in data block, no need to fill, but reserve this point as the start value for next data block. + if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (i < pBlock->info.rows - 1) { + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + } else { + // ignore current row, and do nothing + } + } else { // it is the last row of current block + } + } + pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pSliceInfo->current > pSliceInfo->win.ekey) { @@ -2446,6 +2531,9 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pInfo->fillType = convertFillType(pInterpPhyNode->fillMode); initResultSizeInfo(&pOperator->resultInfo, 4096); + pInfo->pPrevRow = NULL; + pInfo->pNextRow = NULL; + pInfo->pLinearInfo = NULL; pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, (SNodeListNode*)pInterpPhyNode->pFillValues); pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); pInfo->win = pInterpPhyNode->timeRange; From 9692f6f85d79e3ed209fbc808500cecee57679e7 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 8 Aug 2022 16:43:07 +0800 Subject: [PATCH 04/62] build: review readme --- README-CN.md | 1 - README.md | 1 - build.sh | 8 ++++++++ docs/assets/tdengine.svg | 44 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 build.sh create mode 100644 docs/assets/tdengine.svg diff --git a/README-CN.md b/README-CN.md index 39d979c123..942782b109 100644 --- a/README-CN.md +++ b/README-CN.md @@ -187,7 +187,6 @@ git submodule update --init --recursive 这个脚本等价于执行如下命令： ```bash -git submodule update --init --recursive mkdir debug cd debug cmake .. -DBUILD_TOOLS=true diff --git a/README.md b/README.md index d4b8321813..b76a237fab 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,6 @@ You can run the bash script `build.sh` to build both TDengine and taosTools incl It equals to execute following commands: ```bash -git submodule update --init --recursive mkdir debug cd debug cmake .. -DBUILD_TOOLS=true diff --git a/build.sh b/build.sh new file mode 100644 index 0000000000..78f08afa7a --- /dev/null +++ b/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ ! -d debug ]; then + mkdir debug || echo -e "failed to make directory for build" +fi + +cd debug && cmake .. -DBUILD_TOOLS=true && make + diff --git a/docs/assets/tdengine.svg b/docs/assets/tdengine.svg new file mode 100644 index 0000000000..fd3f5955b3 --- /dev/null +++ b/docs/assets/tdengine.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + From d28199019505feba8335d7805bf2dd0cb25e11a1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 8 Aug 2022 18:02:38 +0800 Subject: [PATCH 05/62] fix: fix catalog crash issue --- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/ctgAsync.c | 95 ++++++++++++++++++++++++++-- source/libs/catalog/src/ctgRemote.c | 13 ++-- 3 files changed, 98 insertions(+), 12 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 4ef23860cd..777dcd0592 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -279,6 +279,7 @@ typedef struct SCtgMsgCtx { void* lastOut; void* out; char* target; + SHashObj* pBatchs; } SCtgMsgCtx; @@ -315,7 +316,6 @@ typedef struct SCtgTask { SRWLatch lock; SArray* pParents; SCtgSubRes subRes; - SHashObj* pBatchs; } SCtgTask; typedef struct SCtgTaskReq { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 63d99cc58b..c8165969a5 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -855,6 +855,7 @@ int32_t ctgCallSubCb(SCtgTask *pTask) { int32_t parentNum = taosArrayGetSize(pTask->pParents); for (int32_t i = 0; i < parentNum; ++i) { + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); SCtgTask* pParent = taosArrayGetP(pTask->pParents, i); pParent->subRes.code = pTask->code; @@ -865,7 +866,9 @@ int32_t ctgCallSubCb(SCtgTask *pTask) { } } - pParent->pBatchs = pTask->pBatchs; + SCtgMsgCtx *pParMsgCtx = CTG_GET_TASK_MSGCTX(pParent, -1); + + pParMsgCtx->pBatchs = pMsgCtx->pBatchs; CTG_ERR_JRET(pParent->subRes.fp(pParent)); } @@ -1625,6 +1628,11 @@ _return: int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res)); if (pTask->res) { @@ -1645,6 +1653,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; @@ -1670,7 +1679,11 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); - + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + SCtgTaskReq tReq; tReq.pTask = pTask; tReq.msgIdx = pFetch->fetchIdx; @@ -1686,6 +1699,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgDBCache *dbCache = NULL; SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { @@ -1722,6 +1740,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgDBCache *dbCache = NULL; SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { @@ -1761,6 +1784,7 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbHashsCtx* pCtx = (SCtgTbHashsCtx*)pTask->taskCtx; SCtgDBCache *dbCache = NULL; + SCtgJob* pJob = pTask->pJob; int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; int32_t baseResIdx = 0; @@ -1803,7 +1827,11 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + SBuildUseDBInput input = {0}; strcpy(input.db, pReq->dbFName); @@ -1831,6 +1859,11 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pCtx->pName, &pRes)); if (pRes) { @@ -1852,6 +1885,11 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask *pTask) { SArray* pRes = NULL; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pCtx->pName, dbFName); + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } if (pCtx->tbType <= 0) { CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType)); @@ -1890,6 +1928,11 @@ _return: int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetQnodeListFromMnode(pCtg, pConn, NULL, pTask)); return TSDB_CODE_SUCCESS; @@ -1898,6 +1941,11 @@ int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { int32_t ctgLaunchGetDnodeTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetDnodeListFromMnode(pCtg, pConn, NULL, pTask)); return TSDB_CODE_SUCCESS; @@ -1908,6 +1956,11 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, pCtx->dbFName, NULL, pTask)); @@ -1919,6 +1972,11 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SCtgDBCache *dbCache = NULL; SCtgDbInfoCtx* pCtx = (SCtgDbInfoCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } pTask->res = taosMemoryCalloc(1, sizeof(SDbInfo)); if (NULL == pTask->res) { @@ -1953,6 +2011,11 @@ int32_t ctgLaunchGetIndexTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetIndexInfoFromMnode(pCtg, pConn, pCtx->indexFName, NULL, pTask)); @@ -1963,6 +2026,11 @@ int32_t ctgLaunchGetUdfTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetUdfInfoFromMnode(pCtg, pConn, pCtx->udfName, NULL, pTask)); @@ -1975,6 +2043,11 @@ int32_t ctgLaunchGetUserTask(SCtgTask *pTask) { SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx; bool inCache = false; bool pass = false; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass)); if (inCache) { @@ -1996,6 +2069,11 @@ int32_t ctgLaunchGetUserTask(SCtgTask *pTask) { int32_t ctgLaunchGetSvrVerTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } CTG_ERR_RET(ctgGetSvrVerFromMnode(pCtg, pConn, NULL, pTask)); @@ -2129,7 +2207,10 @@ int32_t ctgSetSubTaskCb(SCtgTask *pSub, SCtgTask *pTask) { if (CTG_TASK_DONE == pSub->status) { pTask->subRes.code = pSub->code; CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].cloneFp)(pSub, &pTask->subRes.res)); - pTask->pBatchs = pSub->pBatchs; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + SCtgMsgCtx *pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + pMsgCtx->pBatchs = pSubMsgCtx->pBatchs; + CTG_ERR_JRET(pTask->subRes.fp(pTask)); } else { if (NULL == pSub->pParents) { @@ -2167,7 +2248,10 @@ int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); if (newTask) { - pSub->pBatchs = pTask->pBatchs; + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + SCtgMsgCtx *pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + pSubMsgCtx->pBatchs = pMsgCtx->pBatchs; + CTG_ERR_RET((*gCtgAsyncFps[pSub->type].launchFp)(pSub)); pSub->status = CTG_TASK_LAUNCHED; } @@ -2180,7 +2264,6 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); - pTask->pBatchs = pJob->pBatchs; qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 45f97865ce..1fdf84e120 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -68,14 +68,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu taskMsg.pData = NULL; taskMsg.len = 0; } - - pTask->pBatchs = pBatchs; SCtgTaskReq tReq; tReq.pTask = pTask; tReq.msgIdx = rsp.msgIdx; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq.msgIdx); + pMsgCtx->pBatchs = pBatchs; - ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1)); + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } @@ -343,7 +343,9 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { ctgError("taosHashInit %d batch failed", CTG_DEFAULT_BATCH_NUM); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - pTask->pBatchs = pBatchs; + + SCtgMsgCtx *pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + pMsgCtx->pBatchs = pBatchs; #endif SCtgTaskReq tReq; @@ -444,7 +446,8 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT uint32_t msgSize) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; - SHashObj* pBatchs = pTask->pBatchs; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SHashObj* pBatchs = pMsgCtx->pBatchs; SCtgJob* pJob = pTask->pJob; SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); SCtgBatch newBatch = {0}; From bf932e1d777331d388b28fc9c77a7d8460cb3784 Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 8 Aug 2022 18:03:52 +0800 Subject: [PATCH 06/62] docs: add tmq sample code --- docs/zh/14-reference/03-connector/java.mdx | 121 ++++++++++++++++----- source/client/src/TSDBJNIConnector.c | 14 +-- 2 files changed, 103 insertions(+), 32 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 88a3674671..d35e44030b 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -670,55 +670,126 @@ public class SchemalessInsertTest { TDengine Java 连接器支持订阅功能，应用 API 如下： -#### 创建订阅 +#### 创建 Topic ```java -TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from meters", false); +Connection connection = DriverManager.getConnection(url, properties); +Statement statement = connection.createStatement(); +statement.executeUpdate("create topic if not exists topic_speed as select ts, speed from speed_table"); ``` `subscribe` 方法的三个参数含义如下： -- topic：订阅的主题（即名称），此参数是订阅的唯一标识 -- sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据 -- restart：如果订阅已经存在，是重新开始，还是继续之前的订阅 +- topic_speed：订阅的主题（即名称），此参数是订阅的唯一标识。 +- sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据。 -如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 +如上面的例子将使用 SQL 语句 `select ts, speed from speed_table` 创建一个名为 `topic_speed` 的订阅。 + +#### 创建 Consumer + +```java +Properties config = new Properties(); +config.setProperty("enable.auto.commit", "true"); +config.setProperty("group.id", "group1"); +config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); + +TaosConsumer consumer = new TaosConsumer<>(config); +``` + +- enable.auto.commit: 是否允许自动提交。 +- group.id: consumer: 所在的 group。 +- value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 #### 订阅消费数据 ```java -int total = 0; while(true) { - TSDBResultSet rs = sub.consume(); - int count = 0; - while(rs.next()) { - count++; - } - total += count; - System.out.printf("%d rows consumed, total %d\n", count, total); - Thread.sleep(1000); + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ResultBean record : records) { + process(record); + } } ``` -`consume` 方法返回一个结果集，其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率（如例子中的 `Thread.sleep(1000)`），否则会给服务端造成不必要的压力。 +`poll` 方法返回一个结果集，其中包含从上次 `poll` 到目前为止的所有新数据。请务必按需选择合理的调用 `poll` 的频率（如例子中的 `Duration.ofMillis(100)`），否则会给服务端造成不必要的压力。 #### 关闭订阅 ```java -sub.close(true); +consumer.close() ``` -`close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。 - -### 关闭资源 +### 使用示例如下： ```java -resultSet.close(); -stmt.close(); -conn.close(); -``` +public abstract class ConsumerLoop { + private final TaosConsumer consumer; + private final List topics; + private final AtomicBoolean shutdown; + private final CountDownLatch shutdownLatch; -> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 + public ConsumerLoop() throws SQLException { + Properties config = new Properties(); + config.setProperty("msg.with.table.name", "true"); + config.setProperty("enable.auto.commit", "true"); + config.setProperty("group.id", "group1"); + config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); + + this.consumer = new TaosConsumer<>(config); + this.topics = Collections.singletonList("topic_speed"); + this.shutdown = new AtomicBoolean(false); + this.shutdownLatch = new CountDownLatch(1); + } + + public abstract void process(ResultBean result); + + public void pollData() throws SQLException { + try { + consumer.subscribe(topics); + + while (!shutdown.get()) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ResultBean record : records) { + process(record); + } + } + } finally { + consumer.close(); + shutdownLatch.countDown(); + } + } + + public void shutdown() throws InterruptedException { + shutdown.set(true); + shutdownLatch.await(); + } + + static class ResultDeserializer extends ReferenceDeserializer { + + } + + static class ResultBean { + private Timestamp ts; + private int speed; + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public int getSpeed() { + return speed; + } + + public void setSpeed(int speed) { + this.speed = speed; + } + } +} +``` ### 与连接池使用 diff --git a/source/client/src/TSDBJNIConnector.c b/source/client/src/TSDBJNIConnector.c index 227c2fff18..38bbec24ce 100644 --- a/source/client/src/TSDBJNIConnector.c +++ b/source/client/src/TSDBJNIConnector.c @@ -757,7 +757,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(J int32_t code = taos_stmt_prepare(pStmt, str, len); taosMemoryFreeClear(str); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("prepareStmt jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -785,7 +785,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI if (code != TSDB_CODE_SUCCESS) { (*env)->ReleaseStringUTFChars(env, jname, name); - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + jniError("bindTableName jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -860,7 +860,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI (*env)->ReleaseStringUTFChars(env, tableName, name); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + jniError("tableNameTags jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); return JNI_TDENGINE_ERROR; } return JNI_SUCCESS; @@ -926,7 +926,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp( taosMemoryFreeClear(b); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("bindColData jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -949,7 +949,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_addBatchImp(JNIEn int32_t code = taos_stmt_add_batch(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("add batch jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -973,7 +973,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(J int32_t code = taos_stmt_execute(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("excute batch jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -997,7 +997,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv int32_t code = taos_stmt_close(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("close stmt jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } From c411b9aebbda77067aab789492ac2f5a1a308dc0 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 8 Aug 2022 18:09:36 +0800 Subject: [PATCH 07/62] fix(stream): memory leak --- include/libs/stream/tstream.h | 35 ++++++- source/dnode/vnode/src/tq/tq.c | 2 + source/libs/stream/src/stream.c | 1 + source/libs/stream/src/streamDispatch.c | 6 ++ source/libs/stream/src/streamExec.c | 8 +- source/libs/stream/src/streamRecover.c | 124 ++++++++++++++++++------ source/libs/stream/src/streamTask.c | 4 +- 7 files changed, 142 insertions(+), 38 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 842e656b9d..0dd137d532 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -226,11 +226,36 @@ typedef struct { int32_t nodeId; int32_t childId; int32_t taskId; - int64_t checkpointVer; - int64_t processedVer; - SEpSet epSet; + // int64_t checkpointVer; + // int64_t processedVer; + SEpSet epSet; } SStreamChildEpInfo; +typedef struct { + int32_t nodeId; + int32_t childId; + int64_t stateSaveVer; + int64_t stateProcessedVer; +} SStreamCheckpointInfo; + +typedef struct { + int64_t streamId; + int64_t checkTs; + int32_t checkpointId; // incremental + int32_t taskId; + SArray* checkpointVer; // SArray +} SStreamMultiVgCheckpointInfo; + +typedef struct { + int32_t taskId; + int32_t checkpointId; // incremental +} SStreamCheckpointKey; + +typedef struct { + int32_t taskId; + SArray* checkpointVer; +} SStreamRecoveringState; + typedef struct SStreamTask { int64_t streamId; int32_t taskId; @@ -256,6 +281,8 @@ typedef struct SStreamTask { // children info SArray* childEpInfo; // SArray + int32_t nextCheckId; + SArray* checkpointInfo; // SArray // exec STaskExec exec; @@ -445,6 +472,7 @@ typedef struct { int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); +void tFreeStreamDispatchReq(SStreamDispatchReq* pReq); int32_t streamSetupTrigger(SStreamTask* pTask); @@ -468,6 +496,7 @@ typedef struct SStreamMeta { TTB* pTaskDb; TTB* pStateDb; SHashObj* pTasks; + SHashObj* pRecoveringState; void* ahandle; TXN txn; FTaskExpand* expandFunc; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c1c680fc56..f7da287ca1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -859,8 +859,10 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { tDecoderInit(&decoder, msgBody, msgLen); if (tDecodeStreamDispatchReq(&decoder, &req) < 0) { code = TSDB_CODE_MSG_DECODE_ERROR; + tDecoderClear(&decoder); goto FAIL; } + tDecoderClear(&decoder); int32_t taskId = req.taskId; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 30f0919cee..e4bf90f308 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -136,6 +136,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); + tFreeStreamDispatchReq(pReq); return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 8d6d31e37f..8b5e6355a2 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -62,6 +62,12 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { return 0; } +void tFreeStreamDispatchReq(SStreamDispatchReq* pReq) { + taosArrayDestroyP(pReq->data, taosMemoryFree); + taosArrayDestroy(pReq->dataLen); + taosMemoryFree(pReq); +} + int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index e662c18a15..4a4d67b89a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -15,7 +15,7 @@ #include "streamInc.h" -static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { +static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { void* exec = pTask->exec.executor; // set input @@ -82,14 +82,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) return 0; } +#if 0 static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); int32_t childId = pBlock->childId; int64_t ver = pBlock->sourceVer; SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); - pChildInfo->processedVer = ver; + /*pChildInfo-> = ver;*/ return 0; } +#endif int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); @@ -198,6 +200,8 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); + streamFreeQitem(data); + if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 2a77ce8a91..28693d15a6 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -87,63 +87,95 @@ int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp return 0; } -typedef struct { - int32_t vgId; - int32_t childId; - int64_t ver; -} SStreamVgVerCheckpoint; - -int32_t tEncodeSStreamVgVerCheckpoint(SEncoder* pEncoder, const SStreamVgVerCheckpoint* pCheckpoint) { - if (tEncodeI32(pEncoder, pCheckpoint->vgId) < 0) return -1; +int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) { + if (tEncodeI32(pEncoder, pCheckpoint->nodeId) < 0) return -1; if (tEncodeI32(pEncoder, pCheckpoint->childId) < 0) return -1; - if (tEncodeI64(pEncoder, pCheckpoint->ver) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->stateProcessedVer) < 0) return -1; return 0; } -int32_t tDecodeSStreamVgVerCheckpoint(SDecoder* pDecoder, SStreamVgVerCheckpoint* pCheckpoint) { - if (tDecodeI32(pDecoder, &pCheckpoint->vgId) < 0) return -1; +int32_t tDecodeSStreamCheckpointInfo(SDecoder* pDecoder, SStreamCheckpointInfo* pCheckpoint) { + if (tDecodeI32(pDecoder, &pCheckpoint->nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pCheckpoint->childId) < 0) return -1; - if (tDecodeI64(pDecoder, &pCheckpoint->ver) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->stateProcessedVer) < 0) return -1; return 0; } -typedef struct { - int64_t streamId; - int64_t checkTs; - int64_t checkpointId; - int32_t taskId; - SArray* checkpointVer; // SArray -} SStreamAggVerCheckpoint; - -int32_t tEncodeSStreamAggVerCheckpoint(SEncoder* pEncoder, const SStreamAggVerCheckpoint* pCheckpoint) { +int32_t tEncodeSStreamMultiVgCheckpointInfo(SEncoder* pEncoder, const SStreamMultiVgCheckpointInfo* pCheckpoint) { if (tEncodeI64(pEncoder, pCheckpoint->streamId) < 0) return -1; if (tEncodeI64(pEncoder, pCheckpoint->checkTs) < 0) return -1; - if (tEncodeI64(pEncoder, pCheckpoint->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->checkpointId) < 0) return -1; if (tEncodeI32(pEncoder, pCheckpoint->taskId) < 0) return -1; int32_t sz = taosArrayGetSize(pCheckpoint->checkpointVer); if (tEncodeI32(pEncoder, sz) < 0) return -1; for (int32_t i = 0; i < sz; i++) { - SStreamVgVerCheckpoint* pOneVgCkpoint = taosArrayGet(pCheckpoint->checkpointVer, i); - if (tEncodeSStreamVgVerCheckpoint(pEncoder, pOneVgCkpoint) < 0) return -1; + SStreamCheckpointInfo* pOneVgCkpoint = taosArrayGet(pCheckpoint->checkpointVer, i); + if (tEncodeSStreamCheckpointInfo(pEncoder, pOneVgCkpoint) < 0) return -1; } return 0; } -int32_t tDecodeSStreamAggVerCheckpoint(SDecoder* pDecoder, SStreamAggVerCheckpoint* pCheckpoint) { +int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCheckpointInfo* pCheckpoint) { if (tDecodeI64(pDecoder, &pCheckpoint->streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pCheckpoint->checkTs) < 0) return -1; - if (tDecodeI64(pDecoder, &pCheckpoint->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->checkpointId) < 0) return -1; if (tDecodeI32(pDecoder, &pCheckpoint->taskId) < 0) return -1; int32_t sz; if (tDecodeI32(pDecoder, &sz) < 0) return -1; for (int32_t i = 0; i < sz; i++) { - SStreamVgVerCheckpoint oneVgCheckpoint; - if (tDecodeSStreamVgVerCheckpoint(pDecoder, &oneVgCheckpoint) < 0) return -1; + SStreamCheckpointInfo oneVgCheckpoint; + if (tDecodeSStreamCheckpointInfo(pDecoder, &oneVgCheckpoint) < 0) return -1; taosArrayPush(pCheckpoint->checkpointVer, &oneVgCheckpoint); } return 0; } +int32_t streamCheckSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + void* buf = NULL; + + ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); + int32_t sz = taosArrayGetSize(pTask->checkpointInfo); + + SStreamMultiVgCheckpointInfo checkpoint; + checkpoint.checkpointId = 0; + checkpoint.checkTs = taosGetTimestampMs(); + checkpoint.streamId = pTask->streamId; + checkpoint.taskId = pTask->taskId; + checkpoint.checkpointVer = pTask->checkpointInfo; + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSStreamMultiVgCheckpointInfo, &checkpoint, len, code); + if (code < 0) { + return -1; + } + + buf = taosMemoryCalloc(1, len); + if (buf == NULL) { + return -1; + } + SEncoder encoder; + tEncoderInit(&encoder, buf, len); + tEncodeSStreamMultiVgCheckpointInfo(&encoder, &checkpoint); + tEncoderClear(&encoder); + + SStreamCheckpointKey key = { + .taskId = pTask->taskId, + .checkpointId = checkpoint.checkpointId, + }; + + if (tdbTbUpsert(pMeta->pStateDb, &key, sizeof(SStreamCheckpointKey), buf, len, &pMeta->txn) < 0) { + ASSERT(0); + goto FAIL; + } + + taosMemoryFree(buf); + return 0; +FAIL: + if (buf) taosMemoryFree(buf); + return -1; +} + int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); // load status @@ -154,9 +186,39 @@ int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { } SDecoder decoder; tDecoderInit(&decoder, pVal, vLen); - SStreamAggVerCheckpoint aggCheckpoint; - tDecodeSStreamAggVerCheckpoint(&decoder, &aggCheckpoint); - /*pTask->*/ + SStreamMultiVgCheckpointInfo aggCheckpoint; + tDecodeSStreamMultiVgCheckpointInfo(&decoder, &aggCheckpoint); + tDecoderClear(&decoder); + + pTask->nextCheckId = aggCheckpoint.checkpointId + 1; + pTask->checkpointInfo = aggCheckpoint.checkpointVer; + + return 0; +} + +int32_t streamCheckAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); + // save and copy state + // save state info + return 0; +} + +int32_t streamRecoverAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); + // try recover sink level + // after all sink level recovered, choose current state backend to recover + return 0; +} + +int32_t streamCheckSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); + // try recover agg level + // + return 0; +} + +int32_t streamRecoverSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 8b5bd849f6..638d39e5cc 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -34,7 +34,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1; - if (tEncodeI64(pEncoder, pInfo->processedVer) < 0) return -1; + /*if (tEncodeI64(pEncoder, pInfo->processedVer) < 0) return -1;*/ if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1; return 0; } @@ -43,7 +43,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1; - if (tDecodeI64(pDecoder, &pInfo->processedVer) < 0) return -1; + /*if (tDecodeI64(pDecoder, &pInfo->processedVer) < 0) return -1;*/ if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1; return 0; } From 63eea61cae1c47f190aaa9208badf16bce1b0a5a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 8 Aug 2022 18:13:35 +0800 Subject: [PATCH 08/62] fix: fix show stables in sysdb result error --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c71292733a..b21e937eb9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2204,10 +2204,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; + char dbName[TSDB_DB_NAME_LEN] = {0}; const char* name = tNameGetTableName(&pInfo->name); if (pInfo->showRewrite) { - char dbName[TSDB_DB_NAME_LEN] = {0}; getDBNameFromCondition(pInfo->pCondition, dbName); sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); } @@ -2217,7 +2217,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTags(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && - IS_SYS_DBNAME(pInfo->req.db)) { + pInfo->showRewrite && IS_SYS_DBNAME(dbName)) { return sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { From 52d317180bc25c7f001bec9aa4c2146b7dc64e2d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 8 Aug 2022 18:32:23 +0800 Subject: [PATCH 09/62] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index b8ae618105..4cf42f9e3b 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -8,9 +8,9 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 从 2.2.0.0 版本开始，TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲解 UDF 的使用方法。 -用户可以通过 UDF 实现两类函数： 标量函数 和 聚合函数。 +用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。 -## 用 C/C++ 语言来定义 UDF +## 接口函数 ### 标量函数 @@ -20,7 +20,7 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 其中 udf 是函数名的占位符，以上述模板实现的函数对行数据块进行标量计算。 -- scalarFunction 中各参数的具体含义是： +- 其中各参数的具体含义是： - inputDataBlock: 输入的数据块 - resultColumn: 输出列 @@ -28,12 +28,12 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 用户可以按照如下函数模板定义自己的聚合函数。 -`int32_t udf_start(SUdfInterBuf *interBuf)` +`int32_t udaf_start(SUdfInterBuf *interBuf)` -`int32_t udf(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` +`int32_t udaf(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` -`int32_t udf_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result)` -其中 udf 是函数名的占位符。其中各参数的具体含义是： +`int32_t udaf_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result)` +其中 udaf 是函数名的占位符。其中各参数的具体含义是： - interBuf：中间结果 buffer。 - inputBlock：输入的数据块。 @@ -48,12 +48,8 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 `int32_t udf_destroy()` -其中 udf 是函数名的占位符。udf_init 完成初始化工作。 udf_destroy 完成清理工作。 +其中 udf 是函数名的占位符，可以替换成自己的函数名。udf_init 完成初始化工作。 udf_destroy 完成清理工作。如果没有初始化工作，无需定义udf_init函数。如果没有清理工作，无需定义udf_destroy函数。 -:::note -如果对应的函数不需要具体的功能，也需要实现一个空函数。 - -::: ### UDF 数据结构 ```c @@ -187,6 +183,8 @@ SELECT X(c1,c2) FROM table/stable; ### 标量函数示例 [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c) +bit_add 实现多列的按位与功能。如果只有一列，返回这一列。bit_add 忽略空值。 +
bit_and.c @@ -198,6 +196,8 @@ SELECT X(c1,c2) FROM table/stable; ### 聚合函数示例 [sqr_sum](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/sqr_sum.c) +sqr_sum 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。 +
sqr_sum.c From c806f5a4554744b971f506e8e0baf4a65420b9af Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 8 Aug 2022 18:43:51 +0800 Subject: [PATCH 10/62] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 4cf42f9e3b..bdb68efef2 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -8,11 +8,11 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 从 2.2.0.0 版本开始，TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲解 UDF 的使用方法。 -用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。 +用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数需要实现以下标量接口函数 udf，聚合函数需要实现 udaf_start ， udaf ， udaf_finish。如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 ## 接口函数 -### 标量函数 +### 标量接口函数 用户可以按照下列函数模板定义自己的标量计算函数 @@ -24,7 +24,7 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 - inputDataBlock: 输入的数据块 - resultColumn: 输出列 -### 聚合函数 +### 聚合接口函数 用户可以按照如下函数模板定义自己的聚合函数。 From c07097ab3f9107f7a11354572a262fd8e0790ec7 Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 8 Aug 2022 18:43:57 +0800 Subject: [PATCH 11/62] docs: add reference --- docs/zh/14-reference/03-connector/java.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index d35e44030b..38c250b110 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -699,6 +699,7 @@ TaosConsumer consumer = new TaosConsumer<>(config); - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 +- 其他参数请参考：[Consumer 参数列表](/develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 From e30b74ad5ab105b1d7794ae976c1172d4dae16f8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 8 Aug 2022 18:20:41 +0800 Subject: [PATCH 12/62] fix(stream): memory leak --- source/dnode/vnode/src/vnd/vnodeSync.c | 6 +++++- source/libs/stream/src/streamDispatch.c | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index a269f81ddd..d819cf92b7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -299,6 +299,10 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vnodePostBlockMsg(pVnode, pMsg); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); + } else { + if (rsp.pCont) { + rpcFreeCont(rsp.pCont); + } } vGTrace("vgId:%d, msg:%p is freed, code:0x%x index:%" PRId64, vgId, pMsg, rsp.code, pMsg->info.conn.applyIndex); @@ -731,4 +735,4 @@ bool vnodeIsLeader(SVnode *pVnode) { } return true; -} \ No newline at end of file +} diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 8b5e6355a2..9d98afa65a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -65,7 +65,6 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { void tFreeStreamDispatchReq(SStreamDispatchReq* pReq) { taosArrayDestroyP(pReq->data, taosMemoryFree); taosArrayDestroy(pReq->dataLen); - taosMemoryFree(pReq); } int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { @@ -285,7 +284,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } code = 0; FAIL_FIXED_DISPATCH: - taosArrayDestroy(req.data); + taosArrayDestroyP(req.data, taosMemoryFree); taosArrayDestroy(req.dataLen); return code; From 48449e80c330a5c89ad4ae93f54363fe24c8b10b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 8 Aug 2022 19:17:28 +0800 Subject: [PATCH 13/62] fix(stream): memory leak --- examples/rust | 1 + source/dnode/mgmt/node_mgmt/src/dmTransport.c | 3 ++- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++++ tools/taos-tools | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) create mode 160000 examples/rust create mode 160000 tools/taos-tools diff --git a/examples/rust b/examples/rust new file mode 160000 index 0000000000..7ed7a97715 --- /dev/null +++ b/examples/rust @@ -0,0 +1 @@ +Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 7efa46c514..a059db6b00 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -255,7 +255,8 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || + msgType == TDMT_SCH_MERGE_FETCH) { return false; } return true; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d819cf92b7..cea0fc0029 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -141,6 +141,10 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { } if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); + } else { + if (rsp.pCont) { + rpcFreeCont(rsp.pCont); + } } } diff --git a/tools/taos-tools b/tools/taos-tools new file mode 160000 index 0000000000..3c7dafeea3 --- /dev/null +++ b/tools/taos-tools @@ -0,0 +1 @@ +Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From df18cc5f47c104097e31dd654a2738bb2251744b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 8 Aug 2022 19:46:37 +0800 Subject: [PATCH 14/62] refactor(sync): speed up replicate --- include/libs/sync/sync.h | 6 ++- include/util/tdef.h | 6 +-- source/dnode/vnode/src/vnd/vnodeSync.c | 3 +- source/libs/sync/inc/syncInt.h | 9 +++- source/libs/sync/inc/syncReplication.h | 2 +- source/libs/sync/src/syncMain.c | 63 ++++++++++++++------------ source/libs/sync/src/syncReplication.c | 37 +++++++++++++-- source/libs/sync/src/syncTimeout.c | 2 +- 8 files changed, 85 insertions(+), 43 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index d96a55c74c..35e79cf45d 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,7 +26,10 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_RESP_TTL_MS 10000000 +#define SYNC_RESP_TTL_MS 10000000 +#define SYNC_SPEED_UP_HB_TIMER 400 +#define SYNC_SPEED_UP_AFTER_MS (1000 * 20) +#define SYNC_SLOW_DOWN_RANGE 100 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 @@ -205,6 +208,7 @@ int32_t syncSetStandby(int64_t rid); ESyncState syncGetMyRole(int64_t rid); bool syncIsReady(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); +bool syncRestoreFinish(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); SyncGroupId syncGetVgId(int64_t rid); void syncGetEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/include/util/tdef.h b/include/util/tdef.h index 6d893765fc..f4e3385651 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -359,11 +359,11 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 -#define TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD 0 +#define TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD (24 * 60 * 60 * 2) #define TSDB_DB_MIN_WAL_RETENTION_SIZE -1 -#define TSDB_DEFAULT_DB_WAL_RETENTION_SIZE 0 +#define TSDB_DEFAULT_DB_WAL_RETENTION_SIZE -1 #define TSDB_DB_MIN_WAL_ROLL_PERIOD 0 -#define TSDB_DEFAULT_DB_WAL_ROLL_PERIOD 0 +#define TSDB_DEFAULT_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1) #define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0 #define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0 diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 50ca81e58b..565175c090 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -722,7 +722,8 @@ void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } bool vnodeIsLeader(SVnode *pVnode) { if (!syncIsReady(pVnode->sync)) { - vDebug("vgId:%d, vnode not ready", pVnode->config.vgId); + vDebug("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncGetMyRoleStr(pVnode->sync), + syncRestoreFinish(pVnode->sync)); return false; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 250b294c19..856870caba 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -162,6 +162,9 @@ typedef struct SSyncNode { // is config changing bool changing; + int64_t startTime; + int64_t lastReplicateTime; + } SSyncNode; // open/close -------------- @@ -186,16 +189,18 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode); // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); + int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); + int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); -int32_t syncNodeStartNowHeartbeatTimer(SSyncNode* pSyncNode); +int32_t syncNodeStartHeartbeatTimerNow(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimerMS(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); -int32_t syncNodeRestartNowHeartbeatTimer(SSyncNode* pSyncNode); +int32_t syncNodeRestartHeartbeatTimerNow(SSyncNode* pSyncNode); int32_t syncNodeRestartNowHeartbeatTimerMS(SSyncNode* pSyncNode, int32_t ms); // utils -------------- diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index dad7d9b5ec..21821be6c7 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -55,7 +55,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode); int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode); -int32_t syncNodeReplicate(SSyncNode* pSyncNode); +int32_t syncNodeReplicate(SSyncNode* pSyncNode, bool isTimer); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); int32_t syncNodeAppendEntriesBatch(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntriesBatch* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5395b72e27..0aa03eae22 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -495,6 +495,18 @@ const char* syncGetMyRoleStr(int64_t rid) { return s; } +bool syncRestoreFinish(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return false; + } + ASSERT(rid == pSyncNode->rid); + bool restoreFinish = pSyncNode->restoreFinish; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return restoreFinish; +} + SyncTerm syncGetMyTerm(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -1086,6 +1098,10 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // start raft // syncNodeBecomeFollower(pSyncNode); + int64_t timeNow = taosGetTimestampMs(); + pSyncNode->startTime = timeNow; + pSyncNode->lastReplicateTime = timeNow; + syncNodeEventLog(pSyncNode, "sync open"); return pSyncNode; @@ -1303,7 +1319,7 @@ int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { return ret; } -int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { +static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncEnvIsStart()) { taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, @@ -1322,26 +1338,21 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { return ret; } -int32_t syncNodeStartHeartbeatTimerMS(SSyncNode* pSyncNode, int32_t ms) { - int32_t ret = 0; - if (syncEnvIsStart()) { - taosTmrReset(pSyncNode->FpHeartbeatTimerCB, ms, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); - atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); - } else { - sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId); - } - - do { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "start heartbeat timer, ms:%d", ms); - syncNodeEventLog(pSyncNode, logBuf); - } while (0); - +int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { + pSyncNode->heartbeatTimerMS = pSyncNode->hbBaseLine; + int32_t ret = syncNodeDoStartHeartbeatTimer(pSyncNode); return ret; } -int32_t syncNodeStartNowHeartbeatTimer(SSyncNode* pSyncNode) { - int32_t ret = syncNodeStartHeartbeatTimerMS(pSyncNode, 1); +int32_t syncNodeStartHeartbeatTimerMS(SSyncNode* pSyncNode, int32_t ms) { + pSyncNode->heartbeatTimerMS = ms; + int32_t ret = syncNodeDoStartHeartbeatTimer(pSyncNode); + return ret; +} + +int32_t syncNodeStartHeartbeatTimerNow(SSyncNode* pSyncNode) { + pSyncNode->heartbeatTimerMS = 1; + int32_t ret = syncNodeDoStartHeartbeatTimer(pSyncNode); return ret; } @@ -1362,9 +1373,9 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { return 0; } -int32_t syncNodeRestartNowHeartbeatTimer(SSyncNode* pSyncNode) { +int32_t syncNodeRestartHeartbeatTimerNow(SSyncNode* pSyncNode) { syncNodeStopHeartbeatTimer(pSyncNode); - syncNodeStartNowHeartbeatTimer(pSyncNode); + syncNodeStartHeartbeatTimerNow(pSyncNode); return 0; } @@ -1942,9 +1953,6 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // Raft 3.6.2 Committing entries from previous terms syncNodeAppendNoop(pSyncNode); -#if 0 // simon - syncNodeReplicate(pSyncNode); -#endif syncMaybeAdvanceCommitIndex(pSyncNode); } else { @@ -2126,9 +2134,6 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { // Raft 3.6.2 Committing entries from previous terms syncNodeAppendNoop(pSyncNode); -#if 0 // simon - syncNodeReplicate(pSyncNode); -#endif syncMaybeAdvanceCommitIndex(pSyncNode); } @@ -2499,7 +2504,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); ASSERT(code == 0); - syncNodeReplicate(ths); + syncNodeReplicate(ths, false); } syncEntryDestory(pEntry); @@ -2572,7 +2577,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI // if mulit replica, start replicate right now if (ths->replicaNum > 1) { - syncNodeReplicate(ths); + syncNodeReplicate(ths, false); // pre commit syncNodePreCommit(ths, pEntry, 0); @@ -2641,7 +2646,7 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p if (ths->replicaNum > 1) { // if multi replica, start replicate right now - syncNodeReplicate(ths); + syncNodeReplicate(ths, false); } else if (ths->replicaNum == 1) { // one replica diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 1a147e391d..29370fd4a5 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -202,17 +202,19 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { syncAppendEntriesBatchDestroy(pMsg); // speed up - if (pMsg->dataCount > 0 && pMsg->prevLogIndex < pSyncNode->commitIndex) { + if (pMsg->dataCount > 0 && pSyncNode->commitIndex - pMsg->prevLogIndex > SYNC_SLOW_DOWN_RANGE) { ret = 1; +#if 0 do { char logBuf[128]; char host[64]; uint16_t port; syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); - snprintf(logBuf, sizeof(logBuf), "speed up for %s:%d, pre-index:%ld", host, port, pMsg->prevLogIndex); + snprintf(logBuf, sizeof(logBuf), "maybe speed up for %s:%d, pre-index:%ld", host, port, pMsg->prevLogIndex); syncNodeEventLog(pSyncNode, logBuf); } while (0); +#endif } } @@ -301,7 +303,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { return ret; } -int32_t syncNodeReplicate(SSyncNode* pSyncNode) { +int32_t syncNodeReplicate(SSyncNode* pSyncNode, bool isTimer) { // start replicate int32_t ret = 0; @@ -323,13 +325,38 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) { break; } - if (ret > 0) { + // start delay + int64_t timeNow = taosGetTimestampMs(); + int64_t startDelay = timeNow - pSyncNode->startTime; + + // replicate delay + int64_t replicateDelay = timeNow - pSyncNode->lastReplicateTime; + pSyncNode->lastReplicateTime = timeNow; + + if (ret > 0 && isTimer && startDelay > SYNC_SPEED_UP_AFTER_MS) { // speed up replicate - int32_t ms = pSyncNode->heartbeatTimerMS < 50 ? pSyncNode->heartbeatTimerMS : 50; + int32_t ms = + pSyncNode->heartbeatTimerMS < SYNC_SPEED_UP_HB_TIMER ? pSyncNode->heartbeatTimerMS : SYNC_SPEED_UP_HB_TIMER; syncNodeRestartNowHeartbeatTimerMS(pSyncNode, ms); +#if 0 + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "replicate speed up"); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); +#endif + } else { syncNodeRestartHeartbeatTimer(pSyncNode); + +#if 0 + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "replicate slow down"); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); +#endif } return ret; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 15b1054e63..65314b812a 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -58,7 +58,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { ++(ths->heartbeatTimerCounter); sInfo("vgId:%d, sync timeout, type:replicate count:%d, heartbeatTimerLogicClockUser:%ld", ths->vgId, ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser); - syncNodeReplicate(ths); + syncNodeReplicate(ths, true); } } else { sError("vgId:%d, unknown timeout-type:%d", ths->vgId, pMsg->timeoutType); From d45ba3f05d9aae3f5564724464701271a83ecfbd Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 8 Aug 2022 20:04:12 +0800 Subject: [PATCH 15/62] build: review readme --- README-CN.md | 7 +------ README.md | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/README-CN.md b/README-CN.md index 942782b109..ad31266bd4 100644 --- a/README-CN.md +++ b/README-CN.md @@ -161,12 +161,7 @@ git clone https://github.com/taosdata/TDengine.git cd TDengine ``` -Go 连接器和 Grafana 插件在其他独立仓库，如果安装它们的话，需要在 TDengine 目录下通过此命令安装： - -```bash -git submodule update --init --recursive -``` - +Go 连接器和 Grafana 插件已移到其他独立仓库。 如果使用 https 协议下载比较慢，可以通过修改 ~/.gitconfig 文件添加以下两行设置使用 ssh 协议下载。需要首先上传 ssh 密钥到 GitHub，详细方法请参考 GitHub 官方文档。 ``` diff --git a/README.md b/README.md index b76a237fab..309bf16d77 100644 --- a/README.md +++ b/README.md @@ -164,12 +164,7 @@ git clone https://github.com/taosdata/TDengine.git cd TDengine ``` -The connectors for go & Grafana and some tools have been moved to separated repositories, -so you should run this command in the TDengine directory to install them: - -```bash -git submodule update --init --recursive -``` +The connectors for go & Grafana and some tools have been moved to separated repositories. You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. From 26648a909f8c9f2a28f18a6baf997588424fccd1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 8 Aug 2022 15:20:12 +0800 Subject: [PATCH 16/62] fix(query): interp + fill(linear) not working TD-18220 --- source/libs/executor/inc/tfill.h | 9 +- source/libs/executor/src/timewindowoperator.c | 112 +++++++++--------- 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index a2beedba36..c2de48d0eb 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -34,9 +34,12 @@ typedef struct SFillColInfo { } SFillColInfo; typedef struct SFillLinearInfo { - SPoint start; - SPoint end; - bool fillLastPoint; + SPoint start; + SPoint end; + bool hasNull; + bool fillLastPoint; + int16_t type; + int32_t bytes; } SFillLinearInfo; typedef struct { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index afc5849963..f532ae54d0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2091,18 +2091,28 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); + SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + int64_t startKey = *(int64_t*)colDataGetData(pTsCol, rowIndex); + int64_t endKey = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); + pLinearInfo->start.key = startKey; + pLinearInfo->end.key = endKey; - pkey->isNull = false; - char* val = colDataGetData(pColInfoData, rowIndex); - memcpy(pkey->pData, val, pkey->bytes); + char* val; + val = colDataGetData(pColInfoData, rowIndex); + memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); + val = colDataGetData(pColInfoData, rowIndex + 1); + memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + + pLinearInfo->hasNull = false; + } else { + pLinearInfo->hasNull = true; } } - pSliceInfo->isNextRowSet = true; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, @@ -2133,52 +2143,36 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataAppend(pDst, rows, (char*)&v, false); + colDataAppend(pDst, rows, (char *)&v, false); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataAppend(pDst, rows, (char*)&v, false); + colDataAppend(pDst, rows, (char *)&v, false); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataAppend(pDst, rows, (char*)&v, false); + colDataAppend(pDst, rows, (char *)&v, false); } pResBlock->info.rows += 1; break; } case TSDB_FILL_LINEAR: { -#if 0 - if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs - || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) { -// goto interp_exit; - } + SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot); - double v1 = -1, v2 = -1; - GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val); - GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val); + SPoint start = pLinearInfo->start; + SPoint end = pLinearInfo->end; + SPoint current = {.key = pSliceInfo->current}; + current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); - SPoint point1 = {.key = ts, .val = &v1}; - SPoint point2 = {.key = nextTs, .val = &v2}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; + if (pLinearInfo->hasNull) { + colDataAppendNULL(pDst, rows); + } else { + taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); + colDataAppend(pDst, rows, (char *)current.val, false); + } - int32_t srcType = pCtx->inputType; - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { - bool exceedMax = false, exceedMin = false; - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin); - if (exceedMax || exceedMin) { - __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0); - if (func(&pCtx->start.val, &pCtx->end.val) <= 0) { - COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val); - } else { - COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val); - } - } - } -#endif - // TODO: pResBlock->info.rows += 1; + pResBlock->info.rows += 1; break; } case TSDB_FILL_PREV: { @@ -2278,13 +2272,16 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - SFillLinearInfo pLinearInfo = {0}; - pLinearInfo.start.key = INT64_MIN; - pLinearInfo.end.key = INT64_MAX; - pLinearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); - pLinearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); - pLinearInfo.fillLastPoint = false; - taosArrayPush(pInfo->pLinearInfo, &pLinearInfo); + SFillLinearInfo linearInfo = {0}; + linearInfo.start.key = INT64_MIN; + linearInfo.end.key = INT64_MAX; + linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); + linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); + linearInfo.hasNull = false; + linearInfo.fillLastPoint = false; + linearInfo.type = pColInfo->info.type; + linearInfo.bytes = pColInfo->info.bytes; + taosArrayPush(pInfo->pLinearInfo, &linearInfo); } return TSDB_CODE_SUCCESS; @@ -2374,6 +2371,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // if its the first point in data block, also fill values between previous(if there's any) and this point; // if its the last point in data block, no need to fill, but reserve this point as the start value for next data block. if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { + doKeepLinearInfo(pSliceInfo, pBlock, i); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { @@ -2395,24 +2395,24 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } else { // it is the last row of current block } - } + } else { // non-linear interpolation + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pResBlock->info.rows >= pResBlock->info.capacity) { - break; + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } } } else if (ts < pSliceInfo->current) { - // in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate + // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); if (i < pBlock->info.rows - 1) { - // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate doKeepNextRows(pSliceInfo, pBlock, i + 1); int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { @@ -2436,7 +2436,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current - // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate doKeepNextRows(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { From ea4904fb40483dc0e3094acec3c3620b5bc10b4d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 8 Aug 2022 20:15:21 +0800 Subject: [PATCH 17/62] fix: fix memory leak issue --- source/client/src/clientHb.c | 8 +++++++- source/client/src/clientImpl.c | 6 +++++- source/client/src/clientMain.c | 5 +++-- source/client/src/clientMsgHandler.c | 2 +- source/libs/qcom/src/queryUtil.c | 2 +- source/libs/scheduler/src/schRemote.c | 5 +++++ source/libs/scheduler/src/schTask.c | 2 +- source/libs/scheduler/src/scheduler.c | 2 ++ 8 files changed, 25 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7031a1ebca..9475d1b51e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,7 +327,13 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { while (pIter != NULL) { int64_t *rid = pIter; SRequestObj *pRequest = acquireRequest(*rid); - if (NULL == pRequest || pRequest->killed) { + if (NULL == pRequest) { + pIter = taosHashIterate(pObj->pRequests, pIter); + continue; + } + + if (pRequest->killed) { + releaseRequest(*rid); pIter = taosHashIterate(pObj->pRequests, pIter); continue; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b85d2a67ac..8cf4e8dc97 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -283,7 +283,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { int32_t code = qExecCommand(pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; @@ -1348,6 +1348,10 @@ int32_t doProcessMsgFromServer(void* param) { } else { memcpy(buf.pData, pMsg->pCont, pMsg->contLen); } + + tscDebug("xxxxx malloc %p, message: %s, size:%d, code: %s, gtid: %s", buf.pData, + TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); + } pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6785390952..8f78fb9bc1 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -182,6 +182,7 @@ void taos_free_result(TAOS_RES *res) { if (TD_RES_QUERY(res)) { SRequestObj *pRequest = (SRequestObj *)res; + tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId); destroyRequest(pRequest); } else if (TD_RES_TMQ(res)) { SMqRspObj *pRsp = (SMqRspObj *)res; @@ -482,7 +483,7 @@ void taos_stop_query(TAOS_RES *res) { int32_t numOfFields = taos_num_fields(pRequest); // It is not a query, no need to stop. if (numOfFields == 0) { - tscDebug("request %" PRIx64 " no need to be killed since not query", pRequest->requestId); + tscDebug("request 0x%" PRIx64 " no need to be killed since not query", pRequest->requestId); return; } @@ -847,7 +848,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { } pRequest->code = - setQueryResultFromRsp(pResultInfo, (SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, false); + setQueryResultFromRsp(pResultInfo, (SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, true); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; pRequest->code = code; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4217cf08b3..0c4cf23c4e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -389,7 +389,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { code = buildShowVariablesRsp(rsp.variables, &pRes); } if (TSDB_CODE_SUCCESS == code) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, false); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true); } tFreeSShowVariablesRsp(&rsp); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4cad6a078b..41333e7756 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -415,7 +415,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } - int32_t metaSize = (pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) * sizeof(SSchema); + int32_t metaSize = sizeof(STableMeta) + (pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 8c9003a9b2..5c993b3831 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -389,6 +389,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { _return: + qDebug("xxxxx free %p", pMsg->pData); taosMemoryFreeClear(pMsg->pData); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, @@ -402,6 +403,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { + qDebug("xxxxx free %p", pMsg->pData); taosMemoryFree(pMsg->pData); } return TSDB_CODE_SUCCESS; @@ -414,6 +416,8 @@ int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("handle %p is broken", pMsg->handle); if (head->isHbParam) { + + qDebug("xxxxx free %p", pMsg->pData); taosMemoryFree(pMsg->pData); SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; @@ -456,6 +460,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { _return: tFreeSSchedulerHbRsp(&rsp); + qDebug("xxxxx free %p", pMsg->pData); taosMemoryFree(pMsg->pData); SCH_RET(code); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index cabca0dc0c..9d87fae62a 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -411,7 +411,7 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 if (pJob->fetched) { SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); - SCH_ERR_RET(rspCode); + SCH_ERR_JRET(rspCode); } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a37cd4fd9e..f1ec7f5e04 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -154,6 +154,8 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { return; } + SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode); + schHandleJobDrop(pJob, errCode); schReleaseJob(*jobId); From a025b4f5a118287b722da82675e028ec86ceac53 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 8 Aug 2022 20:20:05 +0800 Subject: [PATCH 18/62] docs: getting started docker for 3.0 (#15863) --- docs/zh/05-get-started/01-docker.md | 45 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 2edabad3c9..43d13e2864 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -51,35 +51,16 @@ $ taos Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. -Server is Enterprise trial Edition, ver:3.0.0.0 and will expire at 2022-09-24 15:29:46. +Server is Community Edition. taos> ``` - -## 启动 REST 服务 - -taosAdapter 是 TDengine 中提供 REST 服务的组件。下面这条命令会在容器中同时启动 `taosd` 和 `taosadapter` 两个服务组件。 - -```bash -docker run -d --name tdengine-all -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine -``` - -如果想只启动 `taosadapter`： - -```bash -docker run -d --name tdengine-taosa -p 6041-6049:6041-6049 -p 6041-6049:6041-6049/udp -e TAOS_FIRST_EP=tdengine-all tdengine/tdengine:3.0.0.0 taosadapter -``` - -如果想只启动 `taosd`： - -```bash -docker run -d --name tdengine-taosd -p 6030-6042:6030-6042 -p 6030-6042:6030-6042/udp -e TAOS_DISABLE_ADAPTER=true tdengine/tdengine:3.0.0.0 -``` - ## 访问 REST 接口 +taosAdapter 是 TDengine 中提供 REST 服务的组件。下面这条命令会在容器中同时启动 `taosd` 和 `taosadapter` 两个服务组件。默认 Docker 镜像同时启动 TDengine 后台服务 taosd 和 taosAdatper。 + 可以在宿主机使用 curl 通过 RESTful 端口访问 Docker 容器内的 TDengine server。 ``` @@ -96,6 +77,22 @@ curl -L -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql TDengine REST API 详情请参考[官方文档](/reference/rest-api/)。 +## 单独启动 REST 服务 + +如果想只启动 `taosadapter`： + +```bash +docker run -d --network=host --name tdengine-taosa -e TAOS_FIRST_EP=tdengine-taosd tdengine/tdengine:3.0.0.0 taosadapter +``` + +只启动 `taosd`： + +```bash +docker run -d --network=host --name tdengine-taosd -e TAOS_DISABLE_ADAPTER=true tdengine/tdengine:3.0.0.0 +``` + +注意以上为容器使用 host 方式网络配置进行单独部署 taosAdapter 的命令行参数。其他网络访问方式请设置 hostname、 DNS 等必要的网络配置。 + ## 写入数据 可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入。 @@ -107,7 +104,7 @@ TDengine REST API 详情请参考[官方文档](/reference/rest-api/)。 ``` - 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。 + 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"。 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能。 @@ -145,4 +142,4 @@ taos> select avg(current), max(voltage), min(phase) from test.meters where group ```sql taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); -``` \ No newline at end of file +``` From 7aeaa79f4e7f468b5347cb17b374780aa9836225 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 8 Aug 2022 20:28:09 +0800 Subject: [PATCH 19/62] fix(query): fix interp + fill(linear) before inter range issue --- source/libs/executor/src/timewindowoperator.c | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f532ae54d0..f563cf9dc4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2165,6 +2165,11 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SPoint current = {.key = pSliceInfo->current}; current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); + // before interp range, do not fill + if (start.key == INT64_MIN || end.key == INT64_MAX) { + break; + } + if (pLinearInfo->hasNull) { colDataAppendNULL(pDst, rows); } else { @@ -2465,11 +2470,39 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pResBlock->info.rows >= pResBlock->info.capacity) { - break; + if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { + doKeepLinearInfo(pSliceInfo, pBlock, i); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (i < pBlock->info.rows - 1) { + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + } else { + // ignore current row, and do nothing + } + } else { // it is the last row of current block + } + } else { // non-linear interpolation + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } } } From 7d3ac0a2f50aba8a3f3fe526cae63994afb265ca Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 8 Aug 2022 20:45:51 +0800 Subject: [PATCH 20/62] shell: make show same id --- cmake/cmake.define | 4 +--- cmake/cmake.platform | 10 ++++++++++ source/common/src/systable.c | 4 ++-- source/common/src/tglobal.c | 2 +- source/os/src/osFile.c | 4 ++++ source/util/src/tcache.c | 14 +++++++------- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index a33db902ca..92a155ea50 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -115,9 +115,7 @@ ELSE () ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") - IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") - ADD_DEFINITIONS("-D_TD_ARM_") - ELSE () + IF (${TD_INTEL_64} OR ${TD_INTEL_32}) ADD_DEFINITIONS("-msse4.2") IF("${FMA_SUPPORT}" MATCHES "true") MESSAGE(STATUS "turn fma function support on") diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 5c6ffd4b10..49e730a885 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -85,10 +85,14 @@ IF ("${CPUTYPE}" STREQUAL "") MESSAGE(STATUS "The current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") SET(TD_ARM_32 TRUE) + ADD_DEFINITIONS("-D_TD_ARM_") + ADD_DEFINITIONS("-D_TD_ARM_32") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") SET(TD_ARM_64 TRUE) + ADD_DEFINITIONS("-D_TD_ARM_") + ADD_DEFINITIONS("-D_TD_ARM_64") ENDIF () ELSE () # if generate ARM version: @@ -96,15 +100,21 @@ ELSE () IF (${CPUTYPE} MATCHES "aarch32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") + ADD_DEFINITIONS("-D_TD_ARM_") + ADD_DEFINITIONS("-D_TD_ARM_32") SET(TD_ARM_32 TRUE) ELSEIF (${CPUTYPE} MATCHES "aarch64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") + ADD_DEFINITIONS("-D_TD_ARM_") + ADD_DEFINITIONS("-D_TD_ARM_64") SET(TD_ARM_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") MESSAGE(STATUS "input cpuType: mips64") SET(TD_MIPS_64 TRUE) + ADD_DEFINITIONS("-D_TD_MIPS_") + ADD_DEFINITIONS("-D_TD_MIPS_64") ELSEIF (${CPUTYPE} MATCHES "x64") SET(PLATFORM_ARCH_STR "amd64") MESSAGE(STATUS "input cpuType: x64") diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 11faaaad97..2a7c6ca471 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -24,7 +24,7 @@ #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) static const SSysDbTableSchema dnodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, @@ -66,7 +66,7 @@ static const SSysDbTableSchema bnodesSchema[] = { }; static const SSysDbTableSchema clusterSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a0f02d96f9..e01b2ea58e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -423,7 +423,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 2, 1024, 0) != 0) return -1; tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; - tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, TSDB_MAX_MSG_SIZE * 10000L); + tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, 0) != 0) return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index b76769bdb8..6c8e949b25 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -703,7 +703,11 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in int64_t sentbytes; while (leftbytes > 0) { + #ifdef _TD_ARM_32 + sentbytes = sendfile(pFileOut->fd, pFileIn->fd, (long int*)offset, leftbytes); + #else sentbytes = sendfile(pFileOut->fd, pFileIn->fd, offset, leftbytes); + #endif if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { continue; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 7cbc1cd555..dd61f7d225 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -35,7 +35,7 @@ typedef struct SCacheNode { uint64_t addedTime; // the added time when this element is added or updated into cache uint64_t lifespan; // life duration when this element should be remove from cache int64_t expireTime; // expire time - uint64_t signature; + void* signature; struct STrashElem *pTNodeHeader; // point to trash node head uint16_t keyLen : 15; // max key size: 32kb bool inTrashcan : 1; // denote if it is in trash or not @@ -208,7 +208,7 @@ static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force); * @param pNode data node */ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *pNode) { - if (pNode->signature != (uint64_t)pNode) { + if (pNode->signature != pNode) { uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode); return; } @@ -226,7 +226,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode * } static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { - if (pElem->pData->signature != (uint64_t)pElem->pData) { + if (pElem->pData->signature != pElem->pData) { uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); return NULL; } @@ -494,7 +494,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; SCacheNode *ptNode = (SCacheNode *)((char *)data - sizeof(SCacheNode)); - if (ptNode->signature != (uint64_t)ptNode) { + if (ptNode->signature != ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; } @@ -511,7 +511,7 @@ void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) { if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); - if (ptNode->signature != (uint64_t)ptNode) { + if (ptNode->signature != ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; } @@ -539,7 +539,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // It happens when there is only one object in the cache, and two threads which has referenced this object // start to free the it simultaneously [TD-1569]. SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); - if (pNode->signature != (uint64_t)pNode) { + if (pNode->signature != pNode) { uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode); return; } @@ -728,7 +728,7 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); pNewNode->lifespan = duration; pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; - pNewNode->signature = (uint64_t)pNewNode; + pNewNode->signature = pNewNode; pNewNode->size = (uint32_t)sizeInBytes; return pNewNode; From 20fefa6bfd23d7754a2e18a66546606b66d96aff Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 8 Aug 2022 20:56:57 +0800 Subject: [PATCH 21/62] fix: use int32_t when sort merge datablock --- source/libs/parser/src/parInsertData.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae123a3563..7a42d3f403 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// clang-format off +// clang-format on #include "parInsertData.h" #include "catalog.h" @@ -525,7 +525,7 @@ static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* p SBlockRowMerger** ppBlkRowMerger) { SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; STableMeta* pTableMeta = dataBuf->pTableMeta; - int16_t nRows = pBlocks->numOfRows; + int32_t nRows = pBlocks->numOfRows; // size is less than the total size, since duplicated rows may be removed. @@ -546,7 +546,7 @@ static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* p int32_t extendedRowSize = getExtendedRowSize(dataBuf); SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int n = 0; + int32_t n = 0; while (n < nRows) { pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; From e49e9bee6a2465f93c5f355e17e2dc4de3d0d9e8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 8 Aug 2022 20:59:11 +0800 Subject: [PATCH 22/62] other: revert clang format --- source/libs/parser/src/parInsertData.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 7a42d3f403..8fbed3853c 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// clang-format on +// clang-format off #include "parInsertData.h" #include "catalog.h" From e16b2fafbe11f5571f98a7375dfa3d4a72c9d491 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 8 Aug 2022 21:09:25 +0800 Subject: [PATCH 23/62] fix(query): fix interp + fill(linear) start from middle of interpolation range issue --- source/libs/executor/src/timewindowoperator.c | 65 +++++++++++++------ 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f563cf9dc4..0fec1e61d2 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2416,29 +2416,56 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); - if (i < pBlock->info.rows - 1) { - // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate - doKeepNextRows(pSliceInfo, pBlock, i + 1); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pResBlock->info.rows >= pResBlock->info.capacity) { + if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { + doKeepLinearInfo(pSliceInfo, pBlock, i); + //pSliceInfo->current = + // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (i < pBlock->info.rows - 1) { + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); break; } + } else { + // ignore current row, and do nothing } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } else { - // ignore current row, and do nothing + } else { // it is the last row of current block + } + } else { // non-linear interpolation + if (i < pBlock->info.rows - 1) { + // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i + 1); + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + } else { + // ignore current row, and do nothing + } + } else { // it is the last row of current block + doKeepPrevRows(pSliceInfo, pBlock, i); } - } else { // it is the last row of current block - doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate From cb515c6453e02012b5bc0da3423824e2cca8fef9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 8 Aug 2022 21:14:17 +0800 Subject: [PATCH 24/62] refactor(sync): wal default config --- source/libs/parser/src/parAstCreater.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index d19c203ffe..13c2c27efc 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -819,6 +819,10 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE; pOptions->schemaless = TSDB_DEFAULT_DB_SCHEMALESS; + pOptions->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD; + pOptions->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE; + pOptions->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD; + pOptions->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; return (SNode*)pOptions; } From 8661d7532ec5c0b6c218903b0265434514d4fb73 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 8 Aug 2022 21:48:30 +0800 Subject: [PATCH 25/62] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 77 +++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index bdb68efef2..ce8e6731ce 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -10,12 +10,81 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数需要实现以下标量接口函数 udf，聚合函数需要实现 udaf_start ， udaf ， udaf_finish。如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 -## 接口函数 +## 实现标量函数 +假如我们要实现一个名为scalarfn的用户标量函数，函数实现模板如下 +```c +#include "taos.h" +#include "taoserror.h" +#include "taosudf.h" + +// initialization function. if no initialization, we can skip definition of it. The initialization function shall be concatenation of the udf name and _init suffix +// @return error number defined in taoserror.h +int32_t scalarfn_init() { + // initialization. + return TSDB_CODE_SUCCESS; +} + +// scalar function main computation function +// @param inputDataBlock, input data block composed of multiple columns with each column defined by SUdfColumn +// @param resultColumn, output column +// @return error number defined in taoserror.h +int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn* resultColumn) { + // read data from inputDataBlock and process, then output to resultColumn. + return TSDB_CODE_SUCCESS; +} + +// cleanup function. if no cleanup related processing, we can skip definition of it. The destroy function shall be concatenation of the udf name and _destroy suffix. +// @return error number defined in taoserror.h +int32_t scalarfn_destroy() { + // clean up + return TSDB_CODE_SUCCESS; +} +``` +## 实现聚合函数 +假如实现名为aggfn的聚合函数，函数模板如下 +```c +#include "taos.h" +#include "taoserror.h" +#include "taosudf.h" + +// Initialization function. if no initialization, we can skip definition of it. The initialization function shall be concatenation of the udf name and _init suffix +// @return error number defined in taoserror.h +int32_t aggfn_init() { + // initialization. + return TSDB_CODE_SUCCESS; +} + +// aggregate start function. The intermediate value or the state(@interBuf) is initialized in this function. The function name shall be concatenation of udf name and _start suffix +// @return error number defined in taoserror.h +int32_t aggfn_start(SUdfInterBuf* interBuf) { + return TSDB_CODE_SUCESS; +} + +// aggregate reduce function. This function aggregate old state(@interbuf) and one data bock(inputBlock) and output a new state(@newInterBuf). +int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + // read from inputBlock and interBuf and output to newInterBuf + return TSDB_CODE_SUCCESS; +} + +// aggregate function finish function. This function transforms the intermediate value(@interBuf) into the final output(@result). The function name must be concatenation of aggfn and _finish suffix. +// @return error number defined in taoserror.h +int32_t int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result) { + // read data from inputDataBlock and process, then output to resultColumn + return TSDB_CODE_SUCCESS; +} + +// cleanup function. if no cleanup related processing, we can skip definition of it. The destroy function shall be concatenation of the udf name and _destroy suffix. +// @return error number defined in taoserror.h +int32_t aggfn_destroy() { + // clean up + return TSDB_CODE_SUCCESS; +} +``` + +## 接口函数定义 ### 标量接口函数 -用户可以按照下列函数模板定义自己的标量计算函数 - `int32_t udf(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` 其中 udf 是函数名的占位符，以上述模板实现的函数对行数据块进行标量计算。 @@ -26,8 +95,6 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 ### 聚合接口函数 -用户可以按照如下函数模板定义自己的聚合函数。 - `int32_t udaf_start(SUdfInterBuf *interBuf)` `int32_t udaf(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` From cbb9809a4d6fe92ed521a3e7d99959c9b52fa7b3 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 8 Aug 2022 22:53:55 +0800 Subject: [PATCH 26/62] shell: make show same id --- cmake/cmake.define | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 92a155ea50..5639d212d7 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -115,7 +115,7 @@ ELSE () ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") - IF (${TD_INTEL_64} OR ${TD_INTEL_32}) + IF (TD_INTEL_64 OR TD_INTEL_32) ADD_DEFINITIONS("-msse4.2") IF("${FMA_SUPPORT}" MATCHES "true") MESSAGE(STATUS "turn fma function support on") From a504f1ffdd185df318765f99154fe6194b240369 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 9 Aug 2022 09:00:46 +0800 Subject: [PATCH 27/62] fix: remove scl change --- examples/rust | 1 - source/client/src/clientImpl.c | 4 -- source/libs/scalar/inc/sclInt.h | 2 - source/libs/scalar/src/scalar.c | 86 --------------------------- source/libs/scheduler/src/schRemote.c | 5 -- tools/taos-tools | 1 - 6 files changed, 99 deletions(-) delete mode 160000 examples/rust delete mode 160000 tools/taos-tools diff --git a/examples/rust b/examples/rust deleted file mode 160000 index 7ed7a97715..0000000000 --- a/examples/rust +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8cf4e8dc97..4917a01c55 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1348,10 +1348,6 @@ int32_t doProcessMsgFromServer(void* param) { } else { memcpy(buf.pData, pMsg->pCont, pMsg->contLen); } - - tscDebug("xxxxx malloc %p, message: %s, size:%d, code: %s, gtid: %s", buf.pData, - TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); - } pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 36d2c5a49c..d423b92da7 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -45,8 +45,6 @@ typedef struct SScalarCtx { #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) //#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0)) #define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type)) -#define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN) -#define SCL_DOWNGRADE_DATETYPE(_type) ((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT) #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index f08677084d..d0c5a76f4b 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -9,7 +9,6 @@ #include "scalar.h" #include "tudf.h" #include "ttime.h" -#include "tcompare.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type @@ -220,82 +219,6 @@ void sclFreeParamList(SScalarParam *param, int32_t paramNum) { taosMemoryFree(param); } -void sclDowngradeValueType(SValueNode *valueNode) { - switch (valueNode->node.resType.type) { - case TSDB_DATA_TYPE_BIGINT: { - int8_t i8 = valueNode->datum.i; - if (i8 == valueNode->datum.i) { - valueNode->node.resType.type = TSDB_DATA_TYPE_TINYINT; - *(int8_t*)&valueNode->typeData = i8; - break; - } - int16_t i16 = valueNode->datum.i; - if (i16 == valueNode->datum.i) { - valueNode->node.resType.type = TSDB_DATA_TYPE_SMALLINT; - *(int16_t*)&valueNode->typeData = i16; - break; - } - int32_t i32 = valueNode->datum.i; - if (i32 == valueNode->datum.i) { - valueNode->node.resType.type = TSDB_DATA_TYPE_INT; - *(int32_t*)&valueNode->typeData = i32; - break; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT:{ - uint8_t u8 = valueNode->datum.i; - if (u8 == valueNode->datum.i) { - int8_t i8 = valueNode->datum.i; - if (i8 == valueNode->datum.i) { - valueNode->node.resType.type = TSDB_DATA_TYPE_TINYINT; - *(int8_t*)&valueNode->typeData = i8; - } else { - valueNode->node.resType.type = TSDB_DATA_TYPE_UTINYINT; - *(uint8_t*)&valueNode->typeData = u8; - } - break; - } - uint16_t u16 = valueNode->datum.i; - if (u16 == valueNode->datum.i) { - int16_t i16 = valueNode->datum.i; - if (i16 == valueNode->datum.i) { - valueNode->node.resType.type = TSDB_DATA_TYPE_SMALLINT; - *(int16_t*)&valueNode->typeData = i16; - } else { - valueNode->node.resType.type = TSDB_DATA_TYPE_USMALLINT; - *(uint16_t*)&valueNode->typeData = u16; - } - break; - } - uint32_t u32 = valueNode->datum.i; - if (u32 == valueNode->datum.i) { - int32_t i32 = valueNode->datum.i; - if (i32 == valueNode->datum.i) { - valueNode->node.resType.type = TSDB_DATA_TYPE_INT; - *(int32_t*)&valueNode->typeData = i32; - } else { - valueNode->node.resType.type = TSDB_DATA_TYPE_UINT; - *(uint32_t*)&valueNode->typeData = u32; - } - break; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - float f = valueNode->datum.d; - if (FLT_EQUAL(f, valueNode->datum.d)) { - valueNode->node.resType.type = TSDB_DATA_TYPE_FLOAT; - *(float*)&valueNode->typeData = f; - break; - } - break; - } - default: - break; - } -} - int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_LEFT_VALUE: { @@ -308,7 +231,6 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t ASSERT(param->columnData == NULL); param->numOfRows = 1; - /*int32_t code = */sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { colDataAppendNULL(param->columnData, 0); @@ -750,10 +672,6 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } } - - if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { - sclDowngradeValueType(valueNode); - } } if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { @@ -771,10 +689,6 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } } - - if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { - sclDowngradeValueType(valueNode); - } } if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 5c993b3831..8c9003a9b2 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -389,7 +389,6 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { _return: - qDebug("xxxxx free %p", pMsg->pData); taosMemoryFreeClear(pMsg->pData); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, @@ -403,7 +402,6 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { - qDebug("xxxxx free %p", pMsg->pData); taosMemoryFree(pMsg->pData); } return TSDB_CODE_SUCCESS; @@ -416,8 +414,6 @@ int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("handle %p is broken", pMsg->handle); if (head->isHbParam) { - - qDebug("xxxxx free %p", pMsg->pData); taosMemoryFree(pMsg->pData); SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; @@ -460,7 +456,6 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { _return: tFreeSSchedulerHbRsp(&rsp); - qDebug("xxxxx free %p", pMsg->pData); taosMemoryFree(pMsg->pData); SCH_RET(code); } diff --git a/tools/taos-tools b/tools/taos-tools deleted file mode 160000 index 3c7dafeea3..0000000000 --- a/tools/taos-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From 72800f4c87c8b5841ccb463da700ec5d7446a86d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 9 Aug 2022 09:35:26 +0800 Subject: [PATCH 28/62] avoid mem leak --- source/libs/transport/src/transCli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e1df181329..5380cb0481 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1381,6 +1381,7 @@ int transReleaseCliHandle(void* handle) { tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid); if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) { + taosMemoryFree(cmsg); return -1; } return 0; From b80d0e4eca6f5a4eba7ed4493cbbffe5ff915683 Mon Sep 17 00:00:00 2001 From: Yang Zhao Date: Tue, 9 Aug 2022 10:04:44 +0800 Subject: [PATCH 29/62] docs: fix taosbenchmark query sql (#15874) --- docs/zh/05-get-started/01-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 43d13e2864..374a3a79f2 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -129,7 +129,7 @@ taos> select avg(current), max(voltage), min(phase) from test.meters; 查询 location="California.SanFrancisco" 的记录总条数： ```sql -taos> select count(*) from test.meters where location="California.SanFrancisco"; +taos> select count(*) from test.meters where location="San Francisco"; ``` 查询 groupId=10 的所有记录的平均值、最大值、最小值等： From c95b2d80f1041edc0e95d11185cd0dccbf2cf9c1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 9 Aug 2022 10:11:15 +0800 Subject: [PATCH 30/62] Update 01-docker.md (#15875) --- docs/zh/05-get-started/01-docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 374a3a79f2..2a33f0e5a8 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -104,7 +104,7 @@ docker run -d --network=host --name tdengine-taosd -e TAOS_DISABLE_ADAPTER=true ``` - 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"。 + 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"等城市名称。 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能。 @@ -126,7 +126,7 @@ taos> select count(*) from test.meters; taos> select avg(current), max(voltage), min(phase) from test.meters; ``` -查询 location="California.SanFrancisco" 的记录总条数： +查询 location="San Francisco" 的记录总条数： ```sql taos> select count(*) from test.meters where location="San Francisco"; From 134fb8329ed6240be394eb9c868f1c6853ddaf5c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 9 Aug 2022 10:12:07 +0800 Subject: [PATCH 31/62] fix: fix udf document --- docs/zh/07-develop/09-udf.md | 60 ++++++++++++++++++------- tests/script/sh/compile_udf.sh | 2 +- tests/script/sh/{sqr_sum.c => l2norm.c} | 10 ++--- tests/script/tsim/query/udf.sim | 20 ++++----- 4 files changed, 59 insertions(+), 33 deletions(-) rename tests/script/sh/{sqr_sum.c => l2norm.c} (84%) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index ce8e6731ce..528e299330 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -8,10 +8,13 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 从 2.2.0.0 版本开始，TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲解 UDF 的使用方法。 -用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数需要实现以下标量接口函数 udf，聚合函数需要实现 udaf_start ， udaf ， udaf_finish。如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 +用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数对每行数据返回一个值，如求绝对值 abs，正弦函数 sin，字符串拼接函数 concat 等。聚合函数对多行数据进行返回一个值，如求平均数 avg，最大值 max 等。实现udf时，需要实现规定的接口函数。接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。scalarfn，aggfn, udf需要替换成udf函数名。 +- 标量函数需要实现标量接口函数 scalarfn， +- 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 +- 无论标量函数还是聚合函数，如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 ## 实现标量函数 -假如我们要实现一个名为scalarfn的用户标量函数，函数实现模板如下 +标量函数实现模板如下 ```c #include "taos.h" #include "taoserror.h" @@ -40,8 +43,11 @@ int32_t scalarfn_destroy() { return TSDB_CODE_SUCCESS; } ``` +scalarfn 为函数名的占位符，需要替换成函数名，如bit_and。 + ## 实现聚合函数 -假如实现名为aggfn的聚合函数，函数模板如下 + +聚合函数的实现模板如下 ```c #include "taos.h" #include "taoserror.h" @@ -55,21 +61,29 @@ int32_t aggfn_init() { } // aggregate start function. The intermediate value or the state(@interBuf) is initialized in this function. The function name shall be concatenation of udf name and _start suffix +// @param interbuf intermediate value to intialize // @return error number defined in taoserror.h int32_t aggfn_start(SUdfInterBuf* interBuf) { + // initialize intermediate value in interBuf return TSDB_CODE_SUCESS; } // aggregate reduce function. This function aggregate old state(@interbuf) and one data bock(inputBlock) and output a new state(@newInterBuf). +// @param inputBlock input data block +// @param interBuf old state +// @param newInterBuf new state +// @return error number defined in taoserror.h int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { // read from inputBlock and interBuf and output to newInterBuf return TSDB_CODE_SUCCESS; } // aggregate function finish function. This function transforms the intermediate value(@interBuf) into the final output(@result). The function name must be concatenation of aggfn and _finish suffix. +// @interBuf : intermediate value +// @result: final result // @return error number defined in taoserror.h int32_t int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result) { - // read data from inputDataBlock and process, then output to resultColumn + // read data from inputDataBlock and process, then output to result return TSDB_CODE_SUCCESS; } @@ -80,12 +94,17 @@ int32_t aggfn_destroy() { return TSDB_CODE_SUCCESS; } ``` +aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 ## 接口函数定义 +接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。scalarfn，aggfn, udf需要替换成udf函数名。 + +接口函数返回值表示是否成功，如果错误返回错误代码。错误见taoserror.h + ### 标量接口函数 - `int32_t udf(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` + `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` 其中 udf 是函数名的占位符，以上述模板实现的函数对行数据块进行标量计算。 @@ -95,12 +114,12 @@ int32_t aggfn_destroy() { ### 聚合接口函数 -`int32_t udaf_start(SUdfInterBuf *interBuf)` +`int32_t aggfn_start(SUdfInterBuf *interBuf)` -`int32_t udaf(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` +`int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` -`int32_t udaf_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result)` -其中 udaf 是函数名的占位符。其中各参数的具体含义是： +`int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result)` +其中 aggfn 是函数名的占位符。其中各参数的具体含义是： - interBuf：中间结果 buffer。 - inputBlock：输入的数据块。 @@ -108,7 +127,7 @@ int32_t aggfn_destroy() { - result：最终结果。 -其计算过程为：首先调用udf_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个行数据块调用 udf 用数据块更新中间结果，最后再调用 udf_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 +其计算过程为：首先调用aggfn_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个行数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 ### UDF 初始化和销毁 `int32_t udf_init()` @@ -118,7 +137,7 @@ int32_t aggfn_destroy() { 其中 udf 是函数名的占位符，可以替换成自己的函数名。udf_init 完成初始化工作。 udf_destroy 完成清理工作。如果没有初始化工作，无需定义udf_init函数。如果没有清理工作，无需定义udf_destroy函数。 -### UDF 数据结构 +## UDF 数据结构 ```c typedef struct SUdfColumnMeta { int16_t type; @@ -166,6 +185,13 @@ typedef struct SUdfInterBuf { int8_t numOfResult; //zero or one } SUdfInterBuf; ``` +数据结构说明如下： + +- SUdfDataBlock 数据块包含行数 numOfRows 和列数 numCols。udfCols[i] (0 <= i <= numCols-1)表示每一列数据，类型为SUdfColumn*。 +- SUdfColumn 包含列的数据类型定义 colMeta 和列的数据colData。 +- SUdfColumnMeta 成员定义同 taos.h 数据类型定义。 +- SUdfColumnData 数据可以变长，varLenCol定义了变长数据，fixLenCol定义了定长数据。 +- SUdfInterBuf 定义中间结构buffer，以及buffer中结果个数 numOfResult 为了更好的操作以上数据结构，提供了一些便利函数，定义在 taosudf.h。 @@ -214,10 +240,10 @@ CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ - output_type：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。 - 例如，如下语句可以把 libsqrsum.so 创建为系统中可用的 UDF： + 例如，如下语句可以把 libl2norm.so 创建为系统中可用的 UDF： ```sql - CREATE AGGREGATE FUNCTION sqr_sum AS "/home/taos/udf_example/libsqrsum.so" OUTPUTTYPE DOUBLE bufsize 8; + CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 8; ``` ### 管理 UDF @@ -261,15 +287,15 @@ bit_add 实现多列的按位与功能。如果只有一列，返回这一列。
-### 聚合函数示例 [sqr_sum](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/sqr_sum.c) +### 聚合函数示例 [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c) -sqr_sum 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。 +l2norm 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。
-sqr_sum.c +l2norm.c ```c -{{#include tests/script/sh/sqr_sum.c}} +{{#include tests/script/sh/l2norm.c}} ```
diff --git a/tests/script/sh/compile_udf.sh b/tests/script/sh/compile_udf.sh index 5ff3f2bc8a..662c41078a 100755 --- a/tests/script/sh/compile_udf.sh +++ b/tests/script/sh/compile_udf.sh @@ -4,7 +4,7 @@ rm -rf /tmp/udf/libbitand.so /tmp/udf/libsqrsum.so mkdir -p /tmp/udf echo "compile udf bit_and and sqr_sum" gcc -fPIC -shared sh/bit_and.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libbitand.so -gcc -fPIC -shared sh/sqr_sum.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libsqrsum.so +gcc -fPIC -shared sh/sqr_sum.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libl2norm.so echo "debug show /tmp/udf/*.so" ls /tmp/udf/*.so diff --git a/tests/script/sh/sqr_sum.c b/tests/script/sh/l2norm.c similarity index 84% rename from tests/script/sh/sqr_sum.c rename to tests/script/sh/l2norm.c index af57f377ab..8ccdffb8d6 100644 --- a/tests/script/sh/sqr_sum.c +++ b/tests/script/sh/l2norm.c @@ -5,22 +5,22 @@ #include "taosudf.h" -DLL_EXPORT int32_t sqr_sum_init() { +DLL_EXPORT int32_t l2norm_init() { return 0; } -DLL_EXPORT int32_t sqr_sum_destroy() { +DLL_EXPORT int32_t l2norm_destroy() { return 0; } -DLL_EXPORT int32_t sqr_sum_start(SUdfInterBuf *buf) { +DLL_EXPORT int32_t l2norm_start(SUdfInterBuf *buf) { *(int64_t*)(buf->buf) = 0; buf->bufLen = sizeof(double); buf->numOfResult = 0; return 0; } -DLL_EXPORT int32_t sqr_sum(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { +DLL_EXPORT int32_t l2norm(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { double sumSquares = *(double*)interBuf->buf; int8_t numNotNull = 0; for (int32_t i = 0; i < block->numOfCols; ++i) { @@ -67,7 +67,7 @@ DLL_EXPORT int32_t sqr_sum(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInt return 0; } -DLL_EXPORT int32_t sqr_sum_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { +DLL_EXPORT int32_t l2norm_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { if (buf->numOfResult == 0) { resultData->numOfResult = 0; return 0; diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 7259b1e779..2f685f8e24 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -24,10 +24,10 @@ if $system_content == Windows_NT then endi if $system_content == Windows_NT then sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int bufSize 8; - sql create aggregate function sqr_sum as 'C:\\Windows\\Temp\\sqrsum.dll' outputtype double bufSize 8; + sql create aggregate function l2norm as 'C:\\Windows\\Temp\\l2norm.dll' outputtype double bufSize 8; else sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int bufSize 8; - sql create aggregate function sqr_sum as '/tmp/udf/libsqrsum.so' outputtype double bufSize 8; + sql create aggregate function l2norm as '/tmp/udf/libl2norm.so' outputtype double bufSize 8; endi sql show functions; if $rows != 2 then @@ -44,7 +44,7 @@ if $data10 != 2 then return -1 endi -sql select sqr_sum(f) from t; +sql select l2norm(f) from t; if $rows != 1 then print expect 1, actual $rows return -1 @@ -66,7 +66,7 @@ if $data10 != 1 then return -1 endi -sql select sqr_sum(f1, f2) from t2; +sql select l2norm(f1, f2) from t2; if $rows != 1 then return -1 endi @@ -95,7 +95,7 @@ if $data30 != NULL then return -1 endi -sql select sqr_sum(f1, f2) from t2; +sql select l2norm(f1, f2) from t2; print $rows, $data00 if $rows != 1 then return -1 @@ -105,7 +105,7 @@ if $data00 != 2.645751311 then endi sql insert into t2 values(now+4s, 4, 8)(now+5s, 5, 9); -sql select sqr_sum(f1-f2), sqr_sum(f1+f2) from t2; +sql select l2norm(f1-f2), l2norm(f1+f2) from t2; print $rows , $data00 , $data01 if $rows != 1 then return -1; @@ -117,7 +117,7 @@ if $data01 != 18.547236991 then return -1 endi -sql select sqr_sum(bit_and(f2, f1)), sqr_sum(bit_and(f1, f2)) from t2; +sql select l2norm(bit_and(f2, f1)), l2norm(bit_and(f1, f2)) from t2; print $rows , $data00 , $data01 if $rows != 1 then return -1 @@ -129,7 +129,7 @@ if $data01 != 1.414213562 then return -1 endi -sql select sqr_sum(f2) from udf.t2 group by 1-bit_and(f1, f2) order by 1-bit_and(f1,f2); +sql select l2norm(f2) from udf.t2 group by 1-bit_and(f1, f2) order by 1-bit_and(f1,f2); print $rows , $data00 , $data10 , $data20 if $rows != 3 then return -1 @@ -149,10 +149,10 @@ sql show functions; if $rows != 1 then return -1 endi -if $data00 != @sqr_sum@ then +if $data00 != @l2norm@ then return -1 endi -sql drop function sqr_sum; +sql drop function l2norm; sql show functions; if $rows != 0 then return -1 From da80bbc4a2bdf8c14e303a6351d89ef252afff68 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 9 Aug 2022 10:25:32 +0800 Subject: [PATCH 32/62] fix: change sqr_sum udf name to l2norm --- tests/script/sh/compile_udf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/sh/compile_udf.sh b/tests/script/sh/compile_udf.sh index 662c41078a..c7148d7d7d 100755 --- a/tests/script/sh/compile_udf.sh +++ b/tests/script/sh/compile_udf.sh @@ -4,7 +4,7 @@ rm -rf /tmp/udf/libbitand.so /tmp/udf/libsqrsum.so mkdir -p /tmp/udf echo "compile udf bit_and and sqr_sum" gcc -fPIC -shared sh/bit_and.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libbitand.so -gcc -fPIC -shared sh/sqr_sum.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libl2norm.so +gcc -fPIC -shared sh/l2norm.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libl2norm.so echo "debug show /tmp/udf/*.so" ls /tmp/udf/*.so From ad032a0a90cef97b3edc83293399ad1a0588710b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 9 Aug 2022 10:35:39 +0800 Subject: [PATCH 33/62] enh(stream): stream recover --- examples/rust | 1 - include/libs/stream/tstream.h | 1 + source/libs/stream/src/streamExec.c | 18 +++---- source/libs/stream/src/streamMeta.c | 10 ++++ source/libs/stream/src/streamRecover.c | 72 +++++++++++++++++++++----- tools/taos-tools | 1 - 6 files changed, 79 insertions(+), 24 deletions(-) delete mode 160000 examples/rust delete mode 160000 tools/taos-tools diff --git a/examples/rust b/examples/rust deleted file mode 160000 index 7ed7a97715..0000000000 --- a/examples/rust +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 0dd137d532..f5074becec 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -513,6 +513,7 @@ SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaRollBack(SStreamMeta* pMeta); +int32_t streamLoadTasks(SStreamMeta* pMeta); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 4a4d67b89a..7512f792c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -19,23 +19,23 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* void* exec = pTask->exec.executor; // set input - SStreamQueueItem* pItem = (SStreamQueueItem*)data; + const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__GET_RES) { - SStreamTrigger* pTrigger = (SStreamTrigger*)data; + const SStreamTrigger* pTrigger = (const SStreamTrigger*)data; qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data; qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SStreamDataBlock* pBlock = (SStreamDataBlock*)data; - SArray* blocks = pBlock->blocks; + const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; + SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; - SArray* blocks = pMerged->reqs; + const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data; + SArray* blocks = pMerged->reqs; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else { @@ -51,8 +51,8 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; + SSDataBlock block = {0}; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8faa22d643..b74e838628 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -48,8 +48,18 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; + if (streamLoadTasks(pMeta) < 0) { + goto _err; + } return pMeta; + _err: + if (pMeta->path) taosMemoryFree(pMeta->path); + if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); + if (pMeta->pStateDb) tdbTbClose(pMeta->pStateDb); + if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); + if (pMeta->db) tdbClose(pMeta->db); + taosMemoryFree(pMeta); return NULL; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 28693d15a6..2b5af0684d 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -130,14 +130,13 @@ int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCh return 0; } -int32_t streamCheckSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { +int32_t streamSaveStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); - int32_t sz = taosArrayGetSize(pTask->checkpointInfo); SStreamMultiVgCheckpointInfo checkpoint; - checkpoint.checkpointId = 0; + checkpoint.checkpointId = atomic_fetch_add_32(&pTask->nextCheckId, 1); checkpoint.checkTs = taosGetTimestampMs(); checkpoint.streamId = pTask->streamId; checkpoint.taskId = pTask->taskId; @@ -169,16 +168,21 @@ int32_t streamCheckSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { goto FAIL; } + int32_t sz = taosArrayGetSize(pTask->checkpointInfo); + for (int32_t i = 0; i < sz; i++) { + SStreamCheckpointInfo* pCheck = taosArrayGet(pTask->checkpointInfo, i); + pCheck->stateSaveVer = pCheck->stateProcessedVer; + } + taosMemoryFree(buf); return 0; FAIL: if (buf) taosMemoryFree(buf); return -1; + return 0; } -int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); - // load status +int32_t streamLoadStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) { void* pVal = NULL; int32_t vLen = 0; if (tdbTbGet(pMeta->pStateDb, &pTask->taskId, sizeof(void*), &pVal, &vLen) < 0) { @@ -196,29 +200,71 @@ int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -int32_t streamCheckAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { +int32_t streamSaveSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); + return streamSaveStateInfo(pMeta, pTask); +} + +int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); + return streamLoadStateInfo(pMeta, pTask); +} + +int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - // save and copy state + // TODO save and copy state + // save state info + if (streamSaveStateInfo(pMeta, pTask) < 0) { + return -1; + } + return 0; +} + +int32_t streamFetchSinkStatus(SStreamTask* pTask) { + ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); + // set self status to recover_phase1 + // build fetch status msg + // send fetch msg + return 0; +} + +int32_t streamProcessFetchStatusRsp(SStreamMeta* pMeta, SStreamTask* pTask, void* msg) { + // if failed, set timer and retry + // if successful + // add rsp state to partial recover hash + // if complete, begin actual recover return 0; } int32_t streamRecoverAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - // try recover sink level - // after all sink level recovered, choose current state backend to recover + // recover sink level + // after all sink level recovered + // choose suitable state to recover return 0; } -int32_t streamCheckSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { +int32_t streamSaveSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - // try recover agg level - // + // TODO: save and copy state return 0; } int32_t streamRecoverSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); + // if totLevel == 3 + // fetch agg state + // recover from local state to agg state, not send msg + // recover from agg state to most recent log v1 + // enable input queue, set status recover_phase2 + // recover from v1 to queue msg v2, set status normal + + // if totLevel == 2 + // fetch sink state + // recover from local state to sink state v1, send msg + // enable input queue, set status recover_phase2 + // recover from v1 to queue msg v2, set status normal return 0; } diff --git a/tools/taos-tools b/tools/taos-tools deleted file mode 160000 index 3c7dafeea3..0000000000 --- a/tools/taos-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From 9cc08fbb30d2a4083d27da3ba723eee016781f93 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 10:47:00 +0800 Subject: [PATCH 34/62] enh: increase unsynced status when show databases --- source/dnode/mnode/impl/src/mndDb.c | 48 ++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 66f81a3dba..5e6139d9fd 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1495,8 +1495,34 @@ static const char *getCacheModelStr(int8_t cacheModel) { return "unknown"; } -static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, - bool sysDb, ESdbStatus objStatus, bool sysinfo) { +static bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) { + if (pDb->cfg.replications == 1) return true; + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + bool isReady = true; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pDb->uid && pVgroup->replica > 1) { + bool hasLeader = false; + for (int32_t i = 0; i < pVgroup->replica; ++i) { + if (pVgroup->vnodeGid[i].role == TAOS_SYNC_STATE_LEADER) { + hasLeader = true; + } + } + if (!hasLeader) isReady = false; + } + sdbRelease(pSdb, pVgroup); + } + + return isReady; +} + +static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, + int64_t numOfTables, bool sysDb, ESdbStatus objStatus, bool sysinfo) { int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; char *buf = taosMemoryMalloc(bytes); @@ -1509,8 +1535,16 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } char *statusStr = "ready"; - if (objStatus == SDB_STATUS_CREATING) statusStr = "creating"; - if (objStatus == SDB_STATUS_DROPPING) statusStr = "dropping"; + if (objStatus == SDB_STATUS_CREATING) { + statusStr = "creating"; + } else if (objStatus == SDB_STATUS_DROPPING) { + statusStr = "dropping"; + } else { + if (!sysDb && !mndIsDbReady(pMnode, pDb)) { + statusStr = "unsynced"; + } + } + char statusVstr[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusVstr, statusStr, strlen(statusStr)); @@ -1693,7 +1727,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc setInformationSchemaDbCfg(&infoschemaDb); size_t numOfTables = 0; getInfosDbMeta(NULL, &numOfTables); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); + mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); numOfRows += 1; @@ -1701,7 +1735,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc setPerfSchemaDbCfg(&perfschemaDb); numOfTables = 0; getPerfDbMeta(NULL, &numOfTables); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); + mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); numOfRows += 1; pShow->sysDbRsp = true; @@ -1714,7 +1748,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { int32_t numOfTables = 0; sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo); + mndDumpDbInfoData(pMnode, pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo); numOfRows++; } From 81a64d7f1a232724966b47a5a935240f17b0e40b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 9 Aug 2022 11:04:23 +0800 Subject: [PATCH 35/62] change retry code --- source/libs/transport/src/transCli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..2ced2a59dc 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1264,6 +1264,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); cliSchedMsgToNextNode(pMsg, pThrd); return -1; + } else { + pResp->code = TSDB_CODE_APP_NOT_READY; } } } From 38f3cc3e78642e9873b20a4e8720c5ebb2606df3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 9 Aug 2022 11:16:40 +0800 Subject: [PATCH 36/62] fix(query): interp fill(linear) after data range do not fill --- source/libs/executor/src/timewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fec1e61d2..df5cc3324f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2542,7 +2542,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // check if need to interpolate after ts range // except for fill(next) - while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT) { + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); From 39fa900ce5530532d6bab6abccf96490c4166c2f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 11:17:38 +0800 Subject: [PATCH 37/62] enh: wait for the vgroups have leader then create db return --- source/dnode/mnode/impl/inc/mndDb.h | 1 + source/dnode/mnode/impl/inc/mndDef.h | 3 ++- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 24 ++++++++++++++++++++---- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 38a5ecd273..cea0a43b61 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -28,6 +28,7 @@ SDbObj *mndAcquireDb(SMnode *pMnode, const char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); +bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); const char *mndGetDbStr(const char *src); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index c4da9b5c3d..455da6a40e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -162,8 +162,9 @@ typedef struct { int64_t lastExecTime; int32_t lastAction; int32_t lastErrorNo; - tmsg_t lastMsgType; SEpSet lastEpset; + tmsg_t lastMsgType; + tmsg_t originRpcType; char dbname1[TSDB_TABLE_FNAME_LEN]; char dbname2[TSDB_TABLE_FNAME_LEN]; int32_t startFunc; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5e6139d9fd..8edfcad2c6 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1495,7 +1495,7 @@ static const char *getCacheModelStr(int8_t cacheModel) { return "unknown"; } -static bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) { +bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) { if (pDb->cfg.replications == 1) return true; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index b2a0e6aac8..17b4336465 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -124,8 +124,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->oper, _OVER) SDB_SET_INT8(pRaw, dataPos, 0, _OVER) - SDB_SET_INT8(pRaw, dataPos, 0, _OVER) - SDB_SET_INT8(pRaw, dataPos, 0, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->originRpcType, _OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_TABLE_FNAME_LEN, _OVER) @@ -282,13 +281,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &exec, _OVER) SDB_GET_INT8(pRaw, dataPos, &oper, _OVER) SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) - SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) - SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) pTrans->stage = stage; pTrans->policy = policy; pTrans->conflict = conflict; pTrans->exec = exec; pTrans->oper = oper; + SDB_GET_INT16(pRaw, dataPos, &pTrans->originRpcType, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_TABLE_FNAME_LEN, _OVER) @@ -611,6 +609,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, if (pReq != NULL) { taosArrayPush(pTrans->pRpcArray, &pReq->info); + pTrans->originRpcType = pReq->msgType; } mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; @@ -910,6 +909,23 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } } + if (pTrans->originRpcType == TDMT_MND_CREATE_DB) { + mDebug("trans:%d, origin msgtype:%s", pTrans->id, TMSG_INFO(pTrans->originRpcType)); + SDbObj *pDb = mndAcquireDb(pMnode, pTrans->dbname1); + if (pDb != NULL) { + for (int32_t j = 0; j < 12; j++) { + bool ready = mndIsDbReady(pMnode, pDb); + if (!ready) { + mDebug("trans:%d, db:%s not ready yet, wait %d times", pTrans->id, pTrans->dbname1, j); + taosMsleep(1000); + } else { + break; + } + } + } + mndReleaseDb(pMnode, pDb); + } + tmsgSendRsp(&rspMsg); } } From 04a2e8e5414af562054c1162e5e140b0e941fdcc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 11:20:02 +0800 Subject: [PATCH 38/62] fix: the default value of wal-related database parameters is incorrect --- source/libs/parser/src/parAstCreater.c | 8 ++++++++ tests/script/tsim/db/alter_option.sim | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index d19c203ffe..aa526dd440 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -819,6 +819,10 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE; pOptions->schemaless = TSDB_DEFAULT_DB_SCHEMALESS; + pOptions->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD; + pOptions->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE; + pOptions->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD; + pOptions->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; return (SNode*)pOptions; } @@ -846,6 +850,10 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->numOfVgroups = -1; pOptions->singleStable = -1; pOptions->schemaless = -1; + pOptions->walRetentionPeriod = -1; + pOptions->walRetentionSize = -1; + pOptions->walRollPeriod = -1; + pOptions->walSegmentSize = -1; return (SNode*)pOptions; } diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 7df1f02713..00baa9b3f6 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -111,16 +111,16 @@ endi if $data21_db != 1000 then # wal_level fsyncperiod return -1 endi -if $data22_db != 0 then # +if $data22_db != 172800 then # wal_retention_period return -1 endi -if $data23_db != 0 then # +if $data23_db != -1 then # wal_retention_size return -1 endi -if $data24_db != 0 then # +if $data24_db != 86400 then # wal_roll_period return -1 endi -if $data25_db != 0 then # +if $data25_db != 0 then # wal_segment_size return -1 endi From d814473bff1e4412b54c6790e1ce93c60a5f1905 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 9 Aug 2022 11:33:02 +0800 Subject: [PATCH 39/62] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 77 ++++------------------------------- docs/zh/12-taos-sql/26-udf.md | 61 ++++++++++++++++++++------- 2 files changed, 53 insertions(+), 85 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 528e299330..bb2bfada1a 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -6,9 +6,11 @@ description: "支持用户编码的聚合函数和标量函数，在查询中嵌 在有些应用场景中，应用逻辑需要的查询无法直接使用系统内置的函数来表示。利用 UDF 功能，TDengine 可以插入用户编写的处理代码并在查询中使用它们，就能够很方便地解决特殊应用场景中的使用需求。 UDF 通常以数据表中的一列数据做为输入，同时支持以嵌套子查询的结果作为输入。 -从 2.2.0.0 版本开始，TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲解 UDF 的使用方法。 +TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲解 UDF 的使用方法。 -用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数对每行数据返回一个值，如求绝对值 abs，正弦函数 sin，字符串拼接函数 concat 等。聚合函数对多行数据进行返回一个值，如求平均数 avg，最大值 max 等。实现udf时，需要实现规定的接口函数。接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。scalarfn，aggfn, udf需要替换成udf函数名。 +用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数对每行数据返回一个值，如求绝对值 abs，正弦函数 sin，字符串拼接函数 concat 等。聚合函数对多行数据进行返回一个值，如求平均数 avg，最大值 max 等。 + +实现udf时，需要实现规定的接口函数。接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接后。以下列表中的scalarfn，aggfn, udf需要替换成udf函数名。 - 标量函数需要实现标量接口函数 scalarfn， - 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 - 无论标量函数还是聚合函数，如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 @@ -98,9 +100,9 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 ## 接口函数定义 -接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。scalarfn，aggfn, udf需要替换成udf函数名。 +接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。以下描述中函数名称中的 scalarfn，aggfn, udf 需要替换成udf函数名。 -接口函数返回值表示是否成功，如果错误返回错误代码。错误见taoserror.h +接口函数返回值表示是否成功，如果错误返回错误代码，错误代码见taoserror.h。参数类型见数据结构定义。 ### 标量接口函数 @@ -190,7 +192,7 @@ typedef struct SUdfInterBuf { - SUdfDataBlock 数据块包含行数 numOfRows 和列数 numCols。udfCols[i] (0 <= i <= numCols-1)表示每一列数据，类型为SUdfColumn*。 - SUdfColumn 包含列的数据类型定义 colMeta 和列的数据colData。 - SUdfColumnMeta 成员定义同 taos.h 数据类型定义。 -- SUdfColumnData 数据可以变长，varLenCol定义了变长数据，fixLenCol定义了定长数据。 +- SUdfColumnData 数据可以变长，varLenCol定义变长数据，fixLenCol定义定长数据。 - SUdfInterBuf 定义中间结构buffer，以及buffer中结果个数 numOfResult 为了更好的操作以上数据结构，提供了一些便利函数，定义在 taosudf.h。 @@ -207,71 +209,6 @@ gcc -g -O0 -fPIC -shared add_one.c -o add_one.so 这样就准备好了动态链接库 add_one.so 文件，可以供后文创建 UDF 时使用了。为了保证可靠的系统运行，编译器 GCC 推荐使用 7.5 及以上版本。 -## 在系统中管理和使用 UDF - -### 创建 UDF - -用户可以通过 SQL 指令在系统中加载客户端所在主机上的 UDF 函数库（不能通过 RESTful 接口或 HTTP 管理界面来进行这一过程）。一旦创建成功，则当前 TDengine 集群的所有用户都可以在 SQL 指令中使用这些函数。UDF 存储在系统的 MNode 节点上，因此即使重启 TDengine 系统，已经创建的 UDF 也仍然可用。 - -在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 - -- 创建标量函数 -```sql -CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type; -``` - - - function_name：标量函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udf 的实际名称一致； - - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； - - output_type：此函数计算结果的数据类型名称； - - 例如，如下语句可以把 libbitand.so 创建为系统中可用的 UDF： - - ```sql - CREATE FUNCTION bit_and AS "/home/taos/udf_example/libbitand.so" OUTPUTTYPE INT; - ``` - -- 创建聚合函数： -```sql -CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ BUFSIZE buffer_size ]; -``` - - - function_name：聚合函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； - - output_type：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - - buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。 - - 例如，如下语句可以把 libl2norm.so 创建为系统中可用的 UDF： - - ```sql - CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 8; - ``` - -### 管理 UDF - -- 删除指定名称的用户定义函数： -``` -DROP FUNCTION function_name; -``` - -- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如 -```sql -DROP FUNCTION bit_and; -``` -- 显示系统中当前可用的所有 UDF： -```sql -SHOW FUNCTIONS; -``` - -### 调用 UDF - -在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如： -```sql -SELECT X(c1,c2) FROM table/stable; -``` - -表示对名为 c1, c2 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 - - ## 示例代码 ### 标量函数示例 [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c) diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md index 1292206311..7ddcad298b 100644 --- a/docs/zh/12-taos-sql/26-udf.md +++ b/docs/zh/12-taos-sql/26-udf.md @@ -4,34 +4,65 @@ title: 用户自定义函数 --- 除了 TDengine 的内置函数以外，用户还可以编写自己的函数逻辑并加入TDengine系统中。 +## 创建 UDF -## 创建函数 +用户可以通过 SQL 指令在系统中加载客户端所在主机上的 UDF 函数库（不能通过 RESTful 接口或 HTTP 管理界面来进行这一过程）。一旦创建成功，则当前 TDengine 集群的所有用户都可以在 SQL 指令中使用这些函数。UDF 存储在系统的 MNode 节点上，因此即使重启 TDengine 系统，已经创建的 UDF 也仍然可用。 +在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 + +- 创建标量函数 ```sql -CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE buffer_size] +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type; ``` -语法说明： + - function_name：标量函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udf 的实际名称一致； + - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + - output_type：此函数计算结果的数据类型名称； -AGGREGATE：标识此函数是标量函数还是聚集函数。 -func_name：函数名，必须与函数实现中 udf 的实际名称一致。 -library_path：包含UDF函数实现的动态链接库的绝对路径，是在客户端侧主机上的绝对路径。 -type_name：标识此函数的返回类型。 -buffer_size：中间结果的缓冲区大小，单位是字节。不设置则默认为0。 + 例如，如下语句可以把 libbitand.so 创建为系统中可用的 UDF： + ```sql + CREATE FUNCTION bit_and AS "/home/taos/udf_example/libbitand.so" OUTPUTTYPE INT; + ``` + +- 创建聚合函数： +```sql +CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ BUFSIZE buffer_size ]; +``` + + - function_name：聚合函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； + - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + - output_type：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； + - buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。 + + 例如，如下语句可以把 libl2norm.so 创建为系统中可用的 UDF： + + ```sql + CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 8; + ``` 关于如何开发自定义函数，请参考 [UDF使用说明](../../develop/udf)。 -## 删除自定义函数 +## 管理 UDF +- 删除指定名称的用户定义函数： ``` DROP FUNCTION function_name; ``` -- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如 - - -## 显示 UDF - +- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如bit_and, l2norm ```sql -SHOW FUNCTION; +DROP FUNCTION bit_and; ``` +- 显示系统中当前可用的所有 UDF： +```sql +SHOW FUNCTIONS; +``` + +## 调用 UDF + +在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如： +```sql +SELECT X(c1,c2) FROM table/stable; +``` + +表示对名为 c1, c2 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 From 5cf3aaf3636798018bb075ba827b57f0170ab1a3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 9 Aug 2022 11:33:25 +0800 Subject: [PATCH 40/62] docs: getting started docker for 3.0 (#15883) --- docs/zh/05-get-started/01-docker.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 2a33f0e5a8..741f6dfeab 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -10,9 +10,11 @@ title: 通过 Docker 快速体验 TDengine 如果已经安装了 docker， 只需执行下面的命令。 ```shell -docker run -d -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine +docker run -d -p 6030:6030 -p 6041/6041 -p 6043-6049/6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine ``` +注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。 + 确定该容器已经启动并且在正常运行 ```shell From dfe087df743671df2f5e060d5c7f773179e50357 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 9 Aug 2022 11:38:05 +0800 Subject: [PATCH 41/62] modify udf document --- docs/zh/07-develop/09-udf.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index bb2bfada1a..57c5618d54 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -209,6 +209,9 @@ gcc -g -O0 -fPIC -shared add_one.c -o add_one.so 这样就准备好了动态链接库 add_one.so 文件，可以供后文创建 UDF 时使用了。为了保证可靠的系统运行，编译器 GCC 推荐使用 7.5 及以上版本。 +## 管理和使用UDF +关于如何管理和使用UDF，参见[UDF使用说明](../12-taos-sql/26-udf.md) + ## 示例代码 ### 标量函数示例 [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c) From 4503a37b711bffeeb2c9ad2423f44da362ae66d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 11:54:04 +0800 Subject: [PATCH 42/62] enh: adjust the default value of supportVnodes parameters --- source/common/src/tglobal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e01b2ea58e..8d3deff495 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -355,7 +355,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; + + tsNumOfSupportVnodes = tsNumOfCores * 2; + tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; From 45b55f882e5e7e6c5ad3b07d11f77dfe0f7ae2a1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 12:57:27 +0800 Subject: [PATCH 43/62] fix: the default value of wal-related database parameters is incorrect --- source/libs/parser/test/parInitialCTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index b513ff57ed..5cf6c30b31 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -111,6 +111,10 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.numOfVgroups = TSDB_DEFAULT_VN_PER_DB; expect.numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; expect.schemaless = TSDB_DEFAULT_DB_SCHEMALESS; + expect.walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD; + expect.walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE; + expect.walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD; + expect.walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; }; auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; From 01b4e8bb97de4bc7dc1b901ce59dccec27968bf8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 13:35:01 +0800 Subject: [PATCH 44/62] test: set default supportVnodes value --- tests/pytest/util/dnodes-random-fail.py | 3 --- tests/pytest/util/dnodes.py | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 7cadca64a3..9b653844f8 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -24,12 +24,9 @@ class TDSimClient: self.cfgDict = { "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "anyIp": "0", - "sdbDebugFlag": "135", "rpcDebugFlag": "135", "tmrDebugFlag": "131", "cDebugFlag": "135", diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 20e4e4abe6..e530695d1e 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -30,7 +30,6 @@ class TDSimClient: self.path = path self.cfgDict = { "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", @@ -40,6 +39,7 @@ class TDSimClient: "udebugFlag": "143", "jnidebugFlag": "143", "qdebugFlag": "143", + "supportVnodes": "1024", "telemetryReporting": "0", } @@ -117,8 +117,6 @@ class TDDnode: self.valgrind = 0 self.remoteIP = "" self.cfgDict = { - "walLevel": "2", - "fsync": "1000", "monitor": "0", "maxShellConns": "30000", "locale": "en_US.UTF-8", @@ -139,6 +137,7 @@ class TDDnode: "qdebugFlag": "143", "numOfLogLines": "100000000", "statusInterval": "1", + "supportVnodes": "1024", "telemetryReporting": "0" } From aaa588a62f1fd031c00807192c2bc282a200ae3f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 9 Aug 2022 14:10:14 +0800 Subject: [PATCH 45/62] fix: fix stmt parser crash issue --- source/libs/catalog/src/ctgDbg.c | 2 +- source/libs/parser/src/parInsert.c | 7 ++++ source/libs/scheduler/inc/schInt.h | 9 ++++- source/libs/scheduler/src/schFlowCtrl.c | 4 +- source/libs/scheduler/src/schTask.c | 53 ++++++++++++++++++++----- source/util/src/terror.c | 2 +- 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 8333cb28c0..bd3402dc39 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.lockEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 85f73f0663..fa86cfb5b5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1537,6 +1537,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache if (pContext->pStmtCb && *pQuery) { (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj); + if (NULL == context.pVgroupsHashObj) { + context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + } + if (NULL == context.pTableBlockHashObj) { + context.pTableBlockHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } } else { context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); context.pTableBlockHashObj = diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 02e878f4f8..1b3d75f33b 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -61,6 +61,8 @@ typedef enum { #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 +#define SCH_ASYNC_LAUNCH_TASK 0 + typedef struct SSchDebug { bool lockEnable; bool apiEnable; @@ -281,6 +283,11 @@ typedef struct SSchJob { SQueryProfileSummary summary; } SSchJob; +typedef struct SSchTaskCtx { + SSchJob *pJob; + SSchTask *pTask; +} SSchTaskCtx; + extern SSchedulerMgmt schMgmt; #define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) @@ -428,7 +435,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); -int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); int32_t schLaunchFetchTask(SSchJob *pJob); int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction); diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 6b34a394b6..c5c2bfb2bb 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -54,7 +54,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { sum += pTask->plan->execNodeStat.tableNum; } - if (sum < schMgmt.cfg.maxNodeTableNum) { + if (schMgmt.cfg.maxNodeTableNum <= 0 || sum < schMgmt.cfg.maxNodeTableNum) { SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%d", sum); return TSDB_CODE_SUCCESS; } @@ -230,7 +230,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); - SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask)); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); remainNum -= pTask->plan->execNodeStat.tableNum; if (remainNum <= 0) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index cabca0dc0c..729dbf7c1f 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -819,7 +819,10 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { return TSDB_CODE_SUCCESS; } -int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { +int32_t schLaunchTaskImpl(void *param) { + SSchTaskCtx *pCtx = (SSchTaskCtx *)param; + SSchJob *pJob = pCtx->pJob; + SSchTask *pTask = pCtx->pTask; int8_t status = 0; int32_t code = 0; @@ -834,12 +837,12 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status %s", jobTaskStatusStr(status)); - SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) { - SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); + SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC); } @@ -850,19 +853,51 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (TSDB_CODE_SUCCESS != code) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); - SCH_ERR_RET(code); + SCH_ERR_JRET(code); } else { SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } - SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); + SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); + SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); } - SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); + +_return: + + taosMemoryFree(param); + +#if SCH_ASYNC_LAUNCH_TASK + if (code) { + code = schProcessOnTaskFailure(pJob, pTask, code); + } + if (code) { + code = schHandleJobFailure(pJob, code); + } +#endif + + SCH_RET(code); +} + +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { + + SSchTaskCtx *param = taosMemoryCalloc(1, sizeof(SSchTaskCtx)); + if (NULL == param) { + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + param->pJob = pJob; + param->pTask = pTask; + +#if SCH_ASYNC_LAUNCH_TASK + taosAsyncExec(schLaunchTaskImpl, param, NULL); +#else + SCH_ERR_RET(schLaunchTaskImpl(param)); +#endif return TSDB_CODE_SUCCESS; } @@ -878,10 +913,10 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_JRET(schCheckIncTaskFlowQuota(pJob, pTask, &enough)); if (enough) { - SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask)); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); } } else { - SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask)); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); } return TSDB_CODE_SUCCESS; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3c31c893d1..2be1c9f744 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -47,7 +47,7 @@ STaosError errors[] = { // rpc TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Redirect") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Database not ready, need retry") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") From d9a4ea305dc093ae7f345f58b2b4bb3270ec118c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 9 Aug 2022 13:55:04 +0800 Subject: [PATCH 46/62] fix(stream): memory error --- include/libs/stream/tstream.h | 36 +++++++++++++++---------- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/vnode/src/tq/tq.c | 30 ++++++++++++--------- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +++--- source/libs/stream/src/stream.c | 2 ++ source/libs/stream/src/streamExec.c | 7 +++-- source/libs/stream/src/streamRecover.c | 20 +++++++++----- 7 files changed, 66 insertions(+), 39 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index f5074becec..8aacfec397 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -42,8 +42,8 @@ enum { TASK_STATUS__DROPPING, TASK_STATUS__FAIL, TASK_STATUS__STOP, - TASK_STATUS__PREPARE_RECOVER, - TASK_STATUS__RECOVERING, + TASK_STATUS__RECOVER_DOWNSTREAM, + TASK_STATUS__RECOVER_SELF, }; enum { @@ -232,8 +232,8 @@ typedef struct { } SStreamChildEpInfo; typedef struct { - int32_t nodeId; - int32_t childId; + int32_t srcNodeId; + int32_t srcChildId; int64_t stateSaveVer; int64_t stateProcessedVer; } SStreamCheckpointInfo; @@ -394,9 +394,6 @@ typedef struct { int32_t upstreamTaskId; int32_t upstreamChildId; int32_t upstreamNodeId; -#if 0 - int64_t sourceVer; -#endif int32_t blockNum; SArray* dataLen; // SArray SArray* data; // SArray @@ -426,6 +423,7 @@ typedef struct { int32_t rspToTaskId; } SStreamRetrieveRsp; +#if 0 typedef struct { int64_t streamId; int32_t taskId; @@ -462,13 +460,25 @@ int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp); int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp); -typedef struct { - int64_t streamId; -} SPStreamTaskRecoverReq; +int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); +int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); +#endif typedef struct { - int8_t reserved; -} SPStreamTaskRecoverRsp; + int64_t streamId; + int32_t downstreamTaskId; + int32_t taskId; +} SStreamRecoverDownstreamReq; + +typedef struct { + int64_t streamId; + int32_t downstreamTaskId; + int32_t taskId; + SArray* checkpointVer; // SArray +} SStreamRecoverDownstreamRsp; + +int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); +int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, const SStreamRecoverDownstreamRsp* pRsp); int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); @@ -479,8 +489,6 @@ int32_t streamSetupTrigger(SStreamTask* pTask); int32_t streamProcessRunReq(SStreamTask* pTask); int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp); -int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); -int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ba9bb2982f..7bde05aca5 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -504,6 +504,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +#if 0 static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); if (pReq == NULL) { @@ -540,7 +541,6 @@ static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -#if 0 int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { if (pStream->isDistributed) { int32_t lv = taosArrayGetSize(pStream->tasks); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f7da287ca1..02ca248054 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -777,6 +777,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { } } +#if 0 int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; @@ -789,18 +790,6 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { } } -int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { - SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = pRsp->taskId; - SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessDispatchRsp(pTask, pRsp); - return 0; - } else { - return -1; - } -} - int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->rspTaskId; @@ -813,6 +802,19 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { return -1; } } +#endif + +int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t taskId = pRsp->taskId; + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessDispatchRsp(pTask, pRsp); + return 0; + } else { + return -1; + } +} int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; @@ -873,6 +875,8 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { .code = 0, }; streamProcessDispatchReq(pTask, &req, &rsp, false); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); return; } @@ -883,4 +887,6 @@ FAIL: .info = pMsg->info, }; tmsgSendRsp(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fbbe5bc695..0cb66484c0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -332,14 +332,14 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_STREAM_TASK_DISPATCH: // return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, pInfo->workerId != 0); return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); - case TDMT_STREAM_TASK_RECOVER: - return tqProcessTaskRecoverReq(pVnode->pTq, pMsg); + /*case TDMT_STREAM_TASK_RECOVER:*/ + /*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/ case TDMT_STREAM_RETRIEVE: return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); - case TDMT_STREAM_TASK_RECOVER_RSP: - return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg); + /*case TDMT_STREAM_TASK_RECOVER_RSP:*/ + /*return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);*/ case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); default: diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index e4bf90f308..1f8d742de4 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -229,6 +229,7 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { return 0; } +#if 0 int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamTaskRecoverRsp)); ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId); @@ -267,6 +268,7 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) return 0; } +#endif int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { qDebug("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7512f792c1..ffb7c04bf2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -200,14 +200,13 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); - streamFreeQitem(data); - if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { // TODO log failed ver streamQueueProcessFail(pTask->inputQueue); taosArrayDestroy(pRes); + streamFreeQitem(data); return -1; } qRes->type = STREAM_INPUT__DATA_BLOCK; @@ -224,10 +223,14 @@ int32_t streamExecForAll(SStreamTask* pTask) { /*streamQueueProcessFail(pTask->inputQueue);*/ taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosFreeQitem(qRes); + streamFreeQitem(data); return -1; } /*streamQueueProcessSuccess(pTask->inputQueue);*/ + } else { + taosArrayDestroy(pRes); } + streamFreeQitem(data); } return 0; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 2b5af0684d..b2a7e00877 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -15,6 +15,7 @@ #include "streamInc.h" +#if 0 int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -86,17 +87,18 @@ int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp tEndDecode(pDecoder); return 0; } +#endif int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) { - if (tEncodeI32(pEncoder, pCheckpoint->nodeId) < 0) return -1; - if (tEncodeI32(pEncoder, pCheckpoint->childId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->srcChildId) < 0) return -1; if (tEncodeI64(pEncoder, pCheckpoint->stateProcessedVer) < 0) return -1; return 0; } int32_t tDecodeSStreamCheckpointInfo(SDecoder* pDecoder, SStreamCheckpointInfo* pCheckpoint) { - if (tDecodeI32(pDecoder, &pCheckpoint->nodeId) < 0) return -1; - if (tDecodeI32(pDecoder, &pCheckpoint->childId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->srcNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->srcChildId) < 0) return -1; if (tDecodeI64(pDecoder, &pCheckpoint->stateProcessedVer) < 0) return -1; return 0; } @@ -221,11 +223,17 @@ int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -int32_t streamFetchSinkStatus(SStreamTask* pTask) { - ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); +int32_t streamFetchDownstreamStatus(SStreamTask* pTask) { // set self status to recover_phase1 // build fetch status msg // send fetch msg + atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_DOWNSTREAM); + + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else { + ASSERT(0); + } return 0; } From 7f7a2439ff1413486f2244d4bf29b04c0f4ed755 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 14:19:43 +0800 Subject: [PATCH 47/62] enh: adjust the content of the taos sql overview --- docs/zh/12-taos-sql/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/index.md b/docs/zh/12-taos-sql/index.md index cb01b3a918..900fff1ba2 100644 --- a/docs/zh/12-taos-sql/index.md +++ b/docs/zh/12-taos-sql/index.md @@ -5,11 +5,12 @@ description: "TAOS SQL 支持的语法规则、主要查询功能、支持的 SQ 本文档说明 TAOS SQL 支持的语法规则、主要查询功能、支持的 SQL 查询函数，以及常用技巧等内容。阅读本文档需要读者具有基本的 SQL 语言的基础。 -TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 为了便于用户快速上手，在一定程度上提供与标准 SQL 类似的风格和模式。严格意义上，TAOS SQL 并不是也不试图提供标准的 SQL 语法。此外，由于 TDengine 针对的时序性结构化数据不提供删除功能，因此在 TAO SQL 中不提供数据删除的相关功能。 +TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 提供标准的 SQL 语法，并针对时序数据和业务的特点优化和新增了许多语法和功能。TAOS SQL 语句的最大长度为 1M。TAOS SQL 不支持关键字的缩写，例如 DELETE 不能缩写为 DEL。 本章节 SQL 语法遵循如下约定： -- <\> 里的内容是用户需要输入的，但不要输入 <\> 本身 +- 用大写字母表示关键字，但 SQL 本身并不区分关键字和标识符的大小写 +- 用小写字母表示需要用户输入的内容 - \[ \] 表示内容为可选项，但不能输入 [] 本身 - | 表示多选一，选择其中一个即可，但不能输入 | 本身 - … 表示前面的项可重复多个 From 17caf2c6291dbf25d6c7e86f2503f78aa9038d53 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 14:24:27 +0800 Subject: [PATCH 48/62] doc: fix broken links --- docs/zh/14-reference/03-connector/java.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 38c250b110..a7e8c1fa6e 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -358,7 +358,7 @@ JDBC 连接器可能报错的错误码包括 3 种：JDBC driver 本身的报错 具体的错误码请参考： - [TDengine Java Connector](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) -- [TDengine_ERROR_CODE](https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h) +- [TDengine_ERROR_CODE](../error-code) ### 通过参数绑定写入数据 @@ -699,7 +699,7 @@ TaosConsumer consumer = new TaosConsumer<>(config); - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 -- 其他参数请参考：[Consumer 参数列表](/develop/tmq#创建-consumer-以及consumer-group) +- 其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 From 2731434053d5dd302c9f32d3b93619d8e08b0b5c Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 14:25:44 +0800 Subject: [PATCH 49/62] doc: fix broken link --- docs/zh/14-reference/03-connector/java.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index a7e8c1fa6e..c45d670bc6 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -883,7 +883,7 @@ Query OK, 1 row(s) in set (0.000141s) - SpringJdbcTemplate：Spring JdbcTemplate 中使用 taos-jdbcdriver。 - mybatisplus-demo：Springboot + Mybatis 中使用 taos-jdbcdriver。 -请参考：[JDBC example](https://github.com/taosdata/TDengine/tree/develop/examples/JDBC) +请参考：[JDBC example](https://github.com/taosdata/TDengine/tree/3.0/examples/JDBC) ## 最近更新记录 From aace72a06c9146ab82b45d68b8a2bae5a3ae0866 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 9 Aug 2022 14:34:22 +0800 Subject: [PATCH 50/62] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 57c5618d54..08da9e296c 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -10,10 +10,12 @@ TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲 用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数对每行数据返回一个值，如求绝对值 abs，正弦函数 sin，字符串拼接函数 concat 等。聚合函数对多行数据进行返回一个值，如求平均数 avg，最大值 max 等。 -实现udf时，需要实现规定的接口函数。接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接后。以下列表中的scalarfn，aggfn, udf需要替换成udf函数名。 -- 标量函数需要实现标量接口函数 scalarfn， +实现 UDF 时，需要实现规定的接口函数 +- 标量函数需要实现标量接口函数 scalarfn 。 - 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 -- 无论标量函数还是聚合函数，如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 +- 如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 + +接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（_start, _finish, _init, _destroy)的连接。列表中的scalarfn，aggfn, udf需要替换成udf函数名。 ## 实现标量函数 标量函数实现模板如下 @@ -102,17 +104,19 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。以下描述中函数名称中的 scalarfn，aggfn, udf 需要替换成udf函数名。 -接口函数返回值表示是否成功，如果错误返回错误代码，错误代码见taoserror.h。参数类型见数据结构定义。 +接口函数返回值表示是否成功，如果错误返回错误代码，错误代码见taoserror.h。 + +接口函数参数类型见数据结构定义。 ### 标量接口函数 `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` - 其中 udf 是函数名的占位符，以上述模板实现的函数对行数据块进行标量计算。 + 其中 scalarFn 是函数名的占位符。这个函数对数据块进行标量计算，通过设置resultColumn结构体中的变量设置值 -- 其中各参数的具体含义是： +参数的具体含义是： - inputDataBlock: 输入的数据块 - - resultColumn: 输出列 + - resultColumn: 输出列。输出列 ### 聚合接口函数 @@ -121,22 +125,22 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 `int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` `int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result)` -其中 aggfn 是函数名的占位符。其中各参数的具体含义是： +其中 aggfn 是函数名的占位符。首先调用aggfn_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 + +参数的具体含义是： - interBuf：中间结果 buffer。 - inputBlock：输入的数据块。 - newInterBuf：新的中间结果buffer。 - result：最终结果。 -其计算过程为：首先调用aggfn_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个行数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 - ### UDF 初始化和销毁 `int32_t udf_init()` `int32_t udf_destroy()` -其中 udf 是函数名的占位符，可以替换成自己的函数名。udf_init 完成初始化工作。 udf_destroy 完成清理工作。如果没有初始化工作，无需定义udf_init函数。如果没有清理工作，无需定义udf_destroy函数。 +其中 udf 是函数名的占位符。udf_init 完成初始化工作。 udf_destroy 完成清理工作。如果没有初始化工作，无需定义udf_init函数。如果没有清理工作，无需定义udf_destroy函数。 ## UDF 数据结构 @@ -190,10 +194,10 @@ typedef struct SUdfInterBuf { 数据结构说明如下： - SUdfDataBlock 数据块包含行数 numOfRows 和列数 numCols。udfCols[i] (0 <= i <= numCols-1)表示每一列数据，类型为SUdfColumn*。 -- SUdfColumn 包含列的数据类型定义 colMeta 和列的数据colData。 +- SUdfColumn 包含列的数据类型定义 colMeta 和列的数据 colData。 - SUdfColumnMeta 成员定义同 taos.h 数据类型定义。 -- SUdfColumnData 数据可以变长，varLenCol定义变长数据，fixLenCol定义定长数据。 -- SUdfInterBuf 定义中间结构buffer，以及buffer中结果个数 numOfResult +- SUdfColumnData 数据可以变长，varLenCol 定义变长数据，fixLenCol 定义定长数据。 +- SUdfInterBuf 定义中间结构 buffer，以及 buffer 中结果个数 numOfResult 为了更好的操作以上数据结构，提供了一些便利函数，定义在 taosudf.h。 From bec4b2ff19a91e94f73e570a47f15f2b587d7dfe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 9 Aug 2022 15:22:46 +0800 Subject: [PATCH 51/62] change err code --- source/libs/transport/src/transCli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2ced2a59dc..958cf1ddb2 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1265,7 +1265,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { cliSchedMsgToNextNode(pMsg, pThrd); return -1; } else { - pResp->code = TSDB_CODE_APP_NOT_READY; + // change error code for taos client driver if retryCnt exceeds limit + if (0 == strncmp(pTransInst->label, "TSC", strlen("TSC"))) pResp->code = TSDB_CODE_APP_NOT_READY; } } } From e9f06d49c971cc6fdc98330cd9f5eba3485c6b61 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 15:56:17 +0800 Subject: [PATCH 52/62] doc: fix broken links --- docs/zh/14-reference/03-connector/java.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index c45d670bc6..f679e5a272 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -630,7 +630,7 @@ public void setNString(int columnIndex, ArrayList list, int size) throws ### 无模式写入 -从 2.2.0.0 版本开始，TDengine 增加了对无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](/reference/schemaless/)。 +TDengine 支持对无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../reference/schemaless/)。 **注意**： @@ -914,7 +914,7 @@ Query OK, 1 row(s) in set (0.000141s) **解决方法**：重新安装 64 位 JDK。 -4. 其它问题请参考 [FAQ](/train-faq/faq) +4. 其它问题请参考 [FAQ](../../../train-faq/faq) ## API 参考 From a3668eb7ef2d55550357a3356322d22ea0787f25 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 16:05:00 +0800 Subject: [PATCH 53/62] doc: refine java.mdx --- docs/zh/14-reference/03-connector/java.mdx | 50 ++++++++-------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index f679e5a272..8cbe77d29c 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -9,7 +9,7 @@ description: TDengine Java 连接器基于标准 JDBC API 实现, 并提供原 import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -`taos-jdbcdriver` 是 TDengine 的官方 Java 语言连接器，Java 开发人员可以通过它开发存取 TDengine 数据库的应用软件。`taos-jdbcdriver` 实现了 JDBC driver 标准的接口，并提供两种形式的连接器。一种是通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能，一种是通过 taosAdapter 提供的 REST 接口连接 TDengine 实例（2.4.0.0 及更高版本）。REST 连接实现的功能集合和原生连接有少量不同。 +`taos-jdbcdriver` 是 TDengine 的官方 Java 语言连接器，Java 开发人员可以通过它开发存取 TDengine 数据库的应用软件。`taos-jdbcdriver` 实现了 JDBC driver 标准的接口，并提供两种形式的连接器。一种是通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能，一种是通过 taosAdapter 提供的 REST 接口连接 TDengine 实例。REST 连接实现的功能集合和原生连接有少量不同。 ![TDengine Database Connector Java](tdengine-jdbc-connector.webp) @@ -41,19 +41,19 @@ REST 连接支持所有能运行 Java 的平台。 TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： -| TDengine DataType | JDBCType （driver 版本 < 2.0.24） | JDBCType （driver 版本 >= 2.0.24） | -| ----------------- | --------------------------------- | ---------------------------------- | -| TIMESTAMP | java.lang.Long | java.sql.Timestamp | -| INT | java.lang.Integer | java.lang.Integer | -| BIGINT | java.lang.Long | java.lang.Long | -| FLOAT | java.lang.Float | java.lang.Float | -| DOUBLE | java.lang.Double | java.lang.Double | -| SMALLINT | java.lang.Short | java.lang.Short | -| TINYINT | java.lang.Byte | java.lang.Byte | -| BOOL | java.lang.Boolean | java.lang.Boolean | -| BINARY | java.lang.String | byte array | -| NCHAR | java.lang.String | java.lang.String | -| JSON | - | java.lang.String | +| TDengine DataType | JDBCType | +| ----------------- | ---------------------------------- | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | +| SMALLINT | java.lang.Short | +| TINYINT | java.lang.Byte | +| BOOL | java.lang.Boolean | +| BINARY | byte array | +| NCHAR | java.lang.String | +| JSON | java.lang.String | **注意**：JSON 类型仅在 tag 中支持。 @@ -198,7 +198,7 @@ url 中的配置参数如下： - user：登录 TDengine 用户名，默认值 'root'。 - password：用户登录密码，默认值 'taosdata'。 -- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 和 TDengine 2.4.0.12 版本开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 @@ -216,7 +216,7 @@ url 中的配置参数如下： INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); ``` -- 从 taos-jdbcdriver-2.0.36 和 TDengine 2.2.0.0 版本开始，如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); +- 从 taos-jdbcdriver-2.0.36 开始，如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); ::: @@ -362,7 +362,7 @@ JDBC 连接器可能报错的错误码包括 3 种：JDBC driver 本身的报错 ### 通过参数绑定写入数据 -从 2.1.2.0 版本开始，TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 +TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 **注意**： @@ -630,7 +630,7 @@ public void setNString(int columnIndex, ArrayList list, int size) throws ### 无模式写入 -TDengine 支持对无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../reference/schemaless/)。 +TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../reference/schemaless/)。 **注意**： @@ -859,20 +859,6 @@ public static void main(String[] args) throws Exception { > 更多 druid 使用问题请查看[官方说明](https://github.com/alibaba/druid)。 -**注意事项：** - -- TDengine `v1.6.4.1` 版本开始提供了一个专门用于心跳检测的函数 `select server_status()`，所以在使用连接池时推荐使用 `select server_status()` 进行 Validation Query。 - -如下所示，`select server_status()` 执行成功会返回 `1`。 - -```sql -taos> select server_status(); -server_status()| -================ -1 | -Query OK, 1 row(s) in set (0.000141s) -``` - ### 更多示例程序 示例程序源码位于 `TDengine/examples/JDBC` 下: From 1d90aff252f38679b087ad0bf25c6e8028a499f5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 9 Aug 2022 16:13:51 +0800 Subject: [PATCH 54/62] fix(query): fix interp linear interpolation after range issue --- source/libs/executor/src/timewindowoperator.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index df5cc3324f..51fea5ac24 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2395,10 +2395,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } - } else { - // ignore current row, and do nothing } - } else { // it is the last row of current block } } else { // non-linear interpolation pSliceInfo->current = @@ -2418,6 +2415,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { doKeepLinearInfo(pSliceInfo, pBlock, i); + // no need to increate pSliceInfo->current here //pSliceInfo->current = // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { @@ -2436,10 +2434,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } - } else { - // ignore current row, and do nothing } - } else { // it is the last row of current block } } else { // non-linear interpolation if (i < pBlock->info.rows - 1) { @@ -2518,10 +2513,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } - } else { - // ignore current row, and do nothing } - } else { // it is the last row of current block } } else { // non-linear interpolation pSliceInfo->current = @@ -2541,7 +2533,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } // check if need to interpolate after ts range - // except for fill(next) + // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = From 52a620fbbafd03efe0087e6decdcb62c742be3a3 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 9 Aug 2022 16:18:10 +0800 Subject: [PATCH 55/62] Update java.mdx --- docs/zh/14-reference/03-connector/java.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 8cbe77d29c..7a107bd04d 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -630,7 +630,7 @@ public void setNString(int columnIndex, ArrayList list, int size) throws ### 无模式写入 -TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../reference/schemaless/)。 +TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../schemaless)。 **注意**： From 41e2538ade8f83f42ddb7e22e86a17809f9c1a84 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 9 Aug 2022 16:54:59 +0800 Subject: [PATCH 56/62] scalable bloom filter encode & decode --- include/libs/stream/tstreamUpdate.h | 2 + include/util/tbloomfilter.h | 3 + include/util/tscalablebf.h | 3 +- source/libs/stream/src/streamUpdate.c | 108 ++++++++++++++++++ source/libs/stream/test/tstreamUpdateTest.cpp | 107 +++++++++++++++-- source/util/src/tbloomfilter.c | 37 +++++- source/util/src/tscalablebf.c | 42 ++++++- 7 files changed, 285 insertions(+), 17 deletions(-) diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index a4728e6382..78543118da 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -47,6 +47,8 @@ bool updateInfoIgnore(SUpdateInfo *pInfo, STimeWindow* pWin, uint64_t groupId, u void updateInfoDestroy(SUpdateInfo *pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); +int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); +int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); #ifdef __cplusplus } diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index b168da594a..c9ca905f82 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -17,6 +17,7 @@ #define _TD_UTIL_BLOOMFILTER_H_ #include "os.h" +#include "tencode.h" #include "thash.h" #ifdef __cplusplus @@ -42,6 +43,8 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, void tBloomFilterDestroy(SBloomFilter *pBF); void tBloomFilterDump(const SBloomFilter *pBF); bool tBloomFilterIsFull(const SBloomFilter *pBF); +int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder* pEncoder); +SBloomFilter* tBloomFilterDecode(SDecoder* pDecoder); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 8f88f65048..1386f840a8 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -33,7 +33,8 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); void tScalableBfDestroy(SScalableBf *pSBf); -void tScalableBfDump(const SScalableBf *pSBf); +int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder* pEncoder); +SScalableBf* tScalableBfDecode(SDecoder* pDecoder); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index ff1ef7b4b9..c686fa05ce 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -14,6 +14,7 @@ */ #include "tstreamUpdate.h" +#include "tencode.h" #include "ttime.h" #include "query.h" @@ -250,3 +251,110 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { tScalableBfDestroy(pInfo->pCloseWinSBF); pInfo->pCloseWinSBF = NULL; } + +int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { + ASSERT(pInfo); + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + int32_t size = taosArrayGetSize(pInfo->pTsBuckets); + if (tEncodeI32(&encoder, size) < 0) return -1; + for (int32_t i = 0; i < size; i++) { + TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); + if (tEncodeI64(&encoder, *pTs) < 0) return -1; + } + + if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1; + + int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); + if (tEncodeI32(&encoder, sBfSize) < 0) return -1; + for (int32_t i = 0; i < sBfSize; i++) { + SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); + if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; + } + + if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1; + + if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1; + + int32_t mapSize = taosHashGetSize(pInfo->pMap); + if (tEncodeI32(&encoder, mapSize) < 0) return -1; + void* pIte = NULL; + size_t keyLen = 0; + while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1; + if (tEncodeI64(&encoder, *(TSKEY*)pIte) < 0) return -1; + } + + if (tEncodeI64(&encoder, pInfo->scanWindow.skey) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->scanWindow.ekey) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->scanGroupId) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->maxVersion) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { + ASSERT(pInfo); + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + if (tStartDecode(&decoder) < 0) return -1; + + int32_t size = 0; + if (tDecodeI32(&decoder, &size) < 0) return -1; + pInfo->pTsBuckets = taosArrayInit(size, sizeof(TSKEY)); + TSKEY ts = INT64_MIN; + for (int32_t i = 0; i < size; i++) { + if (tDecodeI64(&decoder, &ts) < 0) return -1; + taosArrayPush(pInfo->pTsBuckets, &ts); + } + + if (tDecodeU64(&decoder, &pInfo->numBuckets) < 0) return -1; + + int32_t sBfSize = 0; + if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; + pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); + for (int32_t i = 0; i < sBfSize; i++) { + SScalableBf* pSBf = tScalableBfDecode(&decoder); + if (!pSBf) return -1; + taosArrayPush(pInfo->pTsSBFs, &pSBf); + } + + if (tDecodeU64(&decoder, &pInfo->numSBFs) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; + pInfo->pCloseWinSBF = tScalableBfDecode(&decoder); + + int32_t mapSize = 0; + if (tDecodeI32(&decoder, &mapSize) < 0) return -1; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK); + uint64_t uid = 0; + ts = INT64_MIN; + for(int32_t i = 0; i < mapSize; i++) { + if (tDecodeU64(&decoder, &uid) < 0) return -1; + if (tDecodeI64(&decoder, &ts) < 0) return -1; + taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), &ts, sizeof(TSKEY)); + } + ASSERT(mapSize == taosHashGetSize(pInfo->pMap)); + + if (tDecodeI64(&decoder, &pInfo->scanWindow.skey) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->scanWindow.ekey) < 0) return -1; + if (tDecodeU64(&decoder, &pInfo->scanGroupId) < 0) return -1; + if (tDecodeU64(&decoder, &pInfo->maxVersion) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 93e114db02..933549b8a6 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -6,11 +6,37 @@ using namespace std; #define MAX_NUM_SCALABLE_BF 100000 +bool equalSBF(SScalableBf* left, SScalableBf* right) { + if (left->growth != right->growth) return false; + if (left->numBits != right->numBits) return false; + int lsize = taosArrayGetSize(left->bfArray); + int rsize = taosArrayGetSize(right->bfArray); + if (lsize != rsize) return false; + for (int32_t i = 0; i < lsize; i++) { + SBloomFilter* pLeftBF = (SBloomFilter*)taosArrayGetP(left->bfArray, i); + SBloomFilter* pRightBF = (SBloomFilter*)taosArrayGetP(right->bfArray, i); + if (pLeftBF->errorRate != pRightBF->errorRate) return false; + if (pLeftBF->expectedEntries != pRightBF->expectedEntries) return false; + if (pLeftBF->hashFn1 != pRightBF->hashFn1) return false; + if (pLeftBF->hashFn2 != pRightBF->hashFn2) return false; + if (pLeftBF->hashFunctions != pRightBF->hashFunctions) return false; + if (pLeftBF->numBits != pRightBF->numBits) return false; + if (pLeftBF->numUnits != pRightBF->numUnits) return false; + if (pLeftBF->size != pRightBF->size) return false; + uint64_t* leftUint = (uint64_t*) pLeftBF->buffer; + uint64_t* rightUint = (uint64_t*) pRightBF->buffer; + for (int32_t j = 0; j < pLeftBF->numUnits; j++) { + if (leftUint[j] != rightUint[j]) return false; + } + } + return true; +} + TEST(TD_STREAM_UPDATE_TEST, update) { - int64_t interval = 20 * 1000; - int64_t watermark = 10 * 60 * 1000; + const int64_t interval = 20 * 1000; + const int64_t watermark = 10 * 60 * 1000; SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); - GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,1, 0), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,1, 0), false); GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,1, -1), true); for(int i=0; i < 1024; i++) { @@ -31,15 +57,16 @@ TEST(TD_STREAM_UPDATE_TEST, update) { GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,i, 1), true); } + TSKEY uid = 0; for(int i=3; i < 1024; i++) { - GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,0, i), false); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU, uid, i), false); } - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023); + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU->pMap, &uid, sizeof(uint64_t)), 1023); for(int i=3; i < 1024; i++) { - GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,0, i), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU, uid, i), true); } - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023); + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU->pMap, &uid, sizeof(uint64_t)), 1023); SUpdateInfo *pSU1 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); for(int i=1; i <= watermark / interval; i++) { @@ -75,7 +102,8 @@ TEST(TD_STREAM_UPDATE_TEST, update) { GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU2, 1, i * interval + 5), false); GTEST_ASSERT_EQ(pSU2->minTS, (i-(pSU2->numSBFs-1))*interval); GTEST_ASSERT_EQ(pSU2->numSBFs, watermark / interval); - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU2->pTsBuckets,1), i * interval + 5); + TSKEY uid2 = 1; + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU2->pMap, &uid2, sizeof(uint64_t)), i * interval + 5); } SUpdateInfo *pSU3 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); @@ -84,7 +112,8 @@ TEST(TD_STREAM_UPDATE_TEST, update) { GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU3, i, i * interval + 5 * j), false); GTEST_ASSERT_EQ(pSU3->minTS, 0); GTEST_ASSERT_EQ(pSU3->numSBFs, watermark / interval); - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU3->pTsBuckets, i), i * interval + 5 * j); + uint64_t uid3 = i; + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU3->pMap, &uid3, sizeof(uint64_t)), i * interval + 5 * j); SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU3->pTsSBFs, i); SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0); GTEST_ASSERT_EQ(pBF->size, j); @@ -92,13 +121,66 @@ TEST(TD_STREAM_UPDATE_TEST, update) { } SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1); - GTEST_ASSERT_EQ(pSU4->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval); + GTEST_ASSERT_EQ(pSU4->watermark, pSU4->interval); GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE); SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); - GTEST_ASSERT_EQ(pSU5->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval); + GTEST_ASSERT_EQ(pSU5->watermark, pSU4->interval); GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE); + SUpdateInfo *pSU7 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); + updateInfoAddCloseWindowSBF(pSU7); + for(int64_t i = 1; i < 2048000; i++) { + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,i, i), false); + } + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,100, 1), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,110, 10), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,200, 20), true); + + int32_t bufLen = updateInfoSerialize(NULL, 0, pSU7); + void* buf = taosMemoryCalloc(1, bufLen); + int32_t resSize = updateInfoSerialize(buf, bufLen, pSU7); + + SUpdateInfo *pSU6 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + int32_t desSize = updateInfoDeserialize(buf, bufLen, pSU6); + GTEST_ASSERT_EQ(desSize, 0); + + GTEST_ASSERT_EQ(pSU7->interval, pSU6->interval); + GTEST_ASSERT_EQ(pSU7->maxVersion, pSU6->maxVersion); + GTEST_ASSERT_EQ(pSU7->minTS, pSU6->minTS); + GTEST_ASSERT_EQ(pSU7->numBuckets, pSU6->numBuckets); + GTEST_ASSERT_EQ(pSU7->numSBFs, pSU6->numSBFs); + GTEST_ASSERT_EQ(pSU7->scanGroupId, pSU6->scanGroupId); + GTEST_ASSERT_EQ(pSU7->scanWindow.ekey, pSU6->scanWindow.ekey); + GTEST_ASSERT_EQ(pSU7->scanWindow.skey, pSU6->scanWindow.skey); + GTEST_ASSERT_EQ(pSU7->watermark, pSU6->watermark); + GTEST_ASSERT_EQ(equalSBF(pSU7->pCloseWinSBF, pSU6->pCloseWinSBF), true); + + int32_t mapSize = taosHashGetSize(pSU7->pMap); + GTEST_ASSERT_EQ(mapSize, taosHashGetSize(pSU6->pMap)); + void* pIte = NULL; + size_t keyLen = 0; + while ((pIte = taosHashIterate(pSU7->pMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + void* value6 = taosHashGet(pSU6->pMap, key, keyLen); + GTEST_ASSERT_EQ(*(TSKEY*)pIte, *(TSKEY*)value6); + } + + int32_t buSize = taosArrayGetSize(pSU7->pTsBuckets); + GTEST_ASSERT_EQ(buSize, taosArrayGetSize(pSU6->pTsBuckets)); + for (int32_t i = 0; i < buSize; i++) { + TSKEY ts1 = *(TSKEY*)taosArrayGet(pSU7->pTsBuckets, i); + TSKEY ts2 = *(TSKEY*)taosArrayGet(pSU6->pTsBuckets, i); + GTEST_ASSERT_EQ(ts1, ts2); + } + int32_t lSize = taosArrayGetSize(pSU7->pTsSBFs); + int32_t rSize = taosArrayGetSize(pSU6->pTsSBFs); + GTEST_ASSERT_EQ(lSize, rSize); + for (int32_t i = 0; i < lSize; i++) { + SScalableBf* pLeftSBF = (SScalableBf*)taosArrayGetP(pSU7->pTsSBFs, i); + SScalableBf* pRightSBF = (SScalableBf*)taosArrayGetP(pSU6->pTsSBFs, i); + GTEST_ASSERT_EQ(equalSBF(pLeftSBF, pRightSBF), true); + } updateInfoDestroy(pSU); updateInfoDestroy(pSU1); @@ -106,6 +188,9 @@ TEST(TD_STREAM_UPDATE_TEST, update) { updateInfoDestroy(pSU3); updateInfoDestroy(pSU4); updateInfoDestroy(pSU5); + updateInfoDestroy(pSU6); + updateInfoDestroy(pSU7); + } int main(int argc, char* argv[]) { diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 52c541ae2e..945cb58fcc 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -108,8 +108,41 @@ void tBloomFilterDestroy(SBloomFilter *pBF) { taosMemoryFree(pBF); } -void tBloomFilterDump(const struct SBloomFilter *pBF) { -// ToDo +int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder* pEncoder) { + if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->size) < 0) return -1; + for (uint64_t i = 0; i < pBF->numUnits; i++) { + uint64_t* pUnits = (uint64_t*)pBF->buffer; + if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1; + } + if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1; + return 0; +} + +SBloomFilter* tBloomFilterDecode(SDecoder* pDecoder) { + SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + pBF->buffer = NULL; + if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error; + pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); + for (int32_t i = 0; i < pBF->numUnits; i++) { + uint64_t* pUnits = (uint64_t*)pBF->buffer; + if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; + } + if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; + pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR); + return pBF; + +_error: + tBloomFilterDestroy(pBF); + return NULL; } bool tBloomFilterIsFull(const SBloomFilter *pBF) { diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 9ddac44e20..108eb34803 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -101,6 +101,42 @@ void tScalableBfDestroy(SScalableBf *pSBf) { taosMemoryFree(pSBf); } -void tScalableBfDump(const SScalableBf *pSBf) { - // Todo; -} \ No newline at end of file +int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder* pEncoder) { + if (!pSBf) { + if (tEncodeI32(pEncoder, 0) < 0) return -1; + return 0; + } + int32_t size = taosArrayGetSize(pSBf->bfArray); + if (tEncodeI32(pEncoder, size) < 0) return -1; + for (int32_t i = 0; i < size; i++) { + SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i); + if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1; + } + if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; + if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1; + return 0; +} + +SScalableBf* tScalableBfDecode(SDecoder* pDecoder) { + SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + pSBf->bfArray = NULL; + int32_t size = 0; + if (tDecodeI32(pDecoder, &size) < 0) goto _error; + if (size == 0) { + tScalableBfDestroy(pSBf); + return NULL; + } + pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); + for (int32_t i = 0; i < size; i++) { + SBloomFilter* pBF = tBloomFilterDecode(pDecoder); + if (!pBF) goto _error; + taosArrayPush(pSBf->bfArray, &pBF); + } + if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; + if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; + return pSBf; + +_error: + tScalableBfDestroy(pSBf); + return NULL; +} From 8fb2948136146af1a8633c684cae5daae13443e6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 9 Aug 2022 17:06:49 +0800 Subject: [PATCH 57/62] build: add taosBenchmark to deb --- README-CN.md | 11 ++--------- README.md | 11 ++--------- packaging/deb/makedeb.sh | 2 +- packaging/rpm/tdengine.spec | 2 +- packaging/tools/post.sh | 1 + 5 files changed, 7 insertions(+), 20 deletions(-) diff --git a/README-CN.md b/README-CN.md index ad31266bd4..b2ae777837 100644 --- a/README-CN.md +++ b/README-CN.md @@ -50,19 +50,12 @@ TDengine 目前 2.0 版服务器仅能在 Linux 系统上安装和运行，后 ## 安装工具 -### Ubuntu 16.04 及以上版本 & Debian： +### Ubuntu 18.04 及以上版本 & Debian： ```bash sudo apt-get install -y gcc cmake build-essential git libssl-dev ``` -### Ubuntu 14.04： - -```bash -sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 -export PATH=/usr/lib/binutils-2.26/bin:$PATH -``` - 编译或打包 JDBC 驱动源码，需安装 Java JDK 8 或以上版本和 Apache Maven 2.7 或以上版本。 安装 OpenJDK 8： @@ -89,7 +82,7 @@ taosTools 是用于 TDengine 的辅助工具软件集合。目前它包含 taosB sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config ``` -### CentOS 7： +### CentOS 7.9： ```bash sudo yum install -y gcc gcc-c++ make cmake git openssl-devel diff --git a/README.md b/README.md index 309bf16d77..2f38120b34 100644 --- a/README.md +++ b/README.md @@ -53,19 +53,12 @@ To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in t ## Install build dependencies -### Ubuntu 16.04 and above or Debian +### Ubuntu 18.04 and above or Debian ```bash sudo apt-get install -y gcc cmake build-essential git libssl-dev ``` -### Ubuntu 14.04 - -```bash -sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 -export PATH=/usr/lib/binutils-2.26/bin:$PATH -``` - To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. To install openjdk-8: @@ -91,7 +84,7 @@ To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debia sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config ``` -### CentOS 7 +### CentOS 7.9 ```bash sudo yum install epel-release diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 3e46a1aa55..6de475a4c0 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -60,7 +60,7 @@ cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin -#cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin if [ -f "${compile_dir}/build/bin/taosadapter" ]; then cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||: diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 79e4b2edb8..d3d4bab0e6 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -69,7 +69,7 @@ cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin -#cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin if [ -f %{_compiledir}/build/bin/taosadapter ]; then cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index d637ab8d5a..aa80cfb86c 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -132,6 +132,7 @@ function install_bin() { [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter || : [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosBenchmark || : [ -x ${bin_dir}/TDinsight.sh ] && ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : From 2119997661b8580949b59489fe9c7fd978fe79f2 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 9 Aug 2022 18:17:22 +0800 Subject: [PATCH 58/62] build: add taosBenchmark to deb --- cmake/cmake.platform | 4 ++-- examples/rust | 1 + tools/taos-tools | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) create mode 160000 examples/rust create mode 160000 tools/taos-tools diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 49e730a885..cd1be78fbc 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -97,13 +97,13 @@ IF ("${CPUTYPE}" STREQUAL "") ELSE () # if generate ARM version: # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 - IF (${CPUTYPE} MATCHES "aarch32") + IF (${CPUTYPE} MATCHES "aarch32" or ${CPUTYPE} MATCHES "arm32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") SET(TD_ARM_32 TRUE) - ELSEIF (${CPUTYPE} MATCHES "aarch64") + ELSEIF (${CPUTYPE} MATCHES "aarch64" or ${CPUTYPE} MATCHES "arm64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") ADD_DEFINITIONS("-D_TD_ARM_") diff --git a/examples/rust b/examples/rust new file mode 160000 index 0000000000..7ed7a97715 --- /dev/null +++ b/examples/rust @@ -0,0 +1 @@ +Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/tools/taos-tools b/tools/taos-tools new file mode 160000 index 0000000000..3c7dafeea3 --- /dev/null +++ b/tools/taos-tools @@ -0,0 +1 @@ +Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From 4174add647273a3aa9753c77d9c2c4c19342084c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 9 Aug 2022 18:42:04 +0800 Subject: [PATCH 59/62] chore: udpate libtaos ws submodule for3.0 (#15901) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 * fix: script error handle for 3.0 * chore: update taosws-rs for 3.0 fix segfault * chore: change container_build for websocket build * fix: install script for taosws * fix: . * chore: update taosws-rs for 3.0 * chore: update taosws-rs for 3.0 * chore: update tools/CMakeLists.txt to allow compile taosws-rw on any platform * chore: taosws 648cc62 for 3.0 * chore: update taosws 29424d5 for 3.0 * chore: update cmake/taosws_CMakeLists.txt.in with new repo/commit --- cmake/taosws_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index de6409a8c6..c6d42b686c 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -1,8 +1,8 @@ # taosws-rs ExternalProject_Add(taosws-rs - GIT_REPOSITORY https://github.com/taosdata/taosws-rs.git - GIT_TAG 29424d5 + GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git + GIT_TAG 97c4bac SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From b3b10bbcf69c99fa1648d6f0385bab5fea1b4386 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 16:48:36 +0800 Subject: [PATCH 60/62] test: adjust valgrind case --- tests/script/jenkins/basic.txt | 8 +-- tests/script/tsim/parser/alter1.sim | 2 + .../script/tsim/parser/columnValue_bigint.sim | 2 + tests/script/tsim/parser/columnValue_bool.sim | 1 + .../script/tsim/parser/columnValue_double.sim | 2 + .../script/tsim/parser/columnValue_float.sim | 2 + tests/script/tsim/parser/columnValue_int.sim | 2 + .../tsim/parser/columnValue_smallint.sim | 2 + .../tsim/parser/columnValue_tinyint.sim | 2 + .../script/tsim/parser/columnValue_unsign.sim | 1 + tests/script/tsim/parser/create_db.sim | 4 +- tests/script/tsim/valgrind/basic3.sim | 6 +- tests/script/tsim/valgrind/basic4.sim | 56 +++++++++---------- tests/script/tsim/valgrind/checkError6.sim | 3 + 14 files changed, 53 insertions(+), 40 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0a859b2045..23676153b5 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -84,7 +84,7 @@ ./test.sh -f tsim/insert/update0.sim ./test.sh -f tsim/insert/update1_sort_merge.sim -# ---- parser +# ---- parser ---- ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim @@ -106,7 +106,7 @@ ./test.sh -f tsim/parser/commit.sim ./test.sh -f tsim/parser/condition.sim ./test.sh -f tsim/parser/constCol.sim -#./test.sh -f tsim/parser/create_db.sim +./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim ./test.sh -f tsim/parser/create_tb_with_tag_name.sim ./test.sh -f tsim/parser/create_tb.sim @@ -129,7 +129,7 @@ ./test.sh -f tsim/parser/import.sim ./test.sh -f tsim/parser/insert_multiTbl.sim ./test.sh -f tsim/parser/insert_tb.sim -# TD-17038 ./test.sh -f tsim/parser/interp.sim +# TD-18293 ./test.sh -f tsim/parser/interp.sim ./test.sh -f tsim/parser/join_manyblocks.sim ./test.sh -f tsim/parser/join_multitables.sim ./test.sh -f tsim/parser/join_multivnode.sim @@ -318,7 +318,7 @@ ./test.sh -f tsim/valgrind/checkError7.sim ./test.sh -f tsim/valgrind/checkUdf.sim -# --- vnode +# --- vnode ---- # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim # unsupport ./test.sh -f tsim/vnode/replica3_repeat.sim # unsupport ./test.sh -f tsim/vnode/replica3_vgroup.sim diff --git a/tests/script/tsim/parser/alter1.sim b/tests/script/tsim/parser/alter1.sim index d917f4b61e..6771b35eae 100644 --- a/tests/script/tsim/parser/alter1.sim +++ b/tests/script/tsim/parser/alter1.sim @@ -129,3 +129,5 @@ endi #if $rows != 0 then # return -1 #endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index ae97835dff..a196c61684 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -425,3 +425,5 @@ sql insert into st_bigint_e25 using mt_bigint tags (033) values (now, 00062) #sql_error alter table st_bigint_e23 set tag tagname="abc" #sql_error alter table st_bigint_e24 set tag tagname=" " #sql_error alter table st_bigint_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index d20c4efdc0..ffdb095cee 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -634,3 +634,4 @@ sql alter table st_bool_i4 set tag tagname="abc" sql alter table st_bool_i5 set tag tagname=" " sql alter table st_bool_i6 set tag tagname='' +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_double.sim b/tests/script/tsim/parser/columnValue_double.sim index dae64735ea..d74dda36dc 100644 --- a/tests/script/tsim/parser/columnValue_double.sim +++ b/tests/script/tsim/parser/columnValue_double.sim @@ -528,3 +528,5 @@ sql_error alter table st_double_e22 set tag tagname=abc sql alter table st_double_e23 set tag tagname="abc" sql alter table st_double_e24 set tag tagname=" " sql alter table st_double_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index 9b0ca4b186..a55e1aea2d 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -558,3 +558,5 @@ sql_error alter table st_float_e22 set tag tagname=abc sql alter table st_float_e23 set tag tagname="abc" sql alter table st_float_e24 set tag tagname=" " sql alter table st_float_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 48d95f5ecb..34ad3d93fd 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -423,3 +423,5 @@ sql_error alter table st_int_e22 set tag tagname=abc sql_error alter table st_int_e23 set tag tagname="abc" sql_error alter table st_int_e24 set tag tagname=" " sql alter table st_int_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index ced486ba0b..39608a8efe 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -426,3 +426,5 @@ sql_error alter table st_smallint_e22 set tag tagname=abc sql_error alter table st_smallint_e23 set tag tagname="abc" sql_error alter table st_smallint_e24 set tag tagname=" " sql alter table st_smallint_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index bc1fcd3445..fe2734c245 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -424,3 +424,5 @@ sql_error alter table st_tinyint_e22 set tag tagname=abc sql_error alter table st_tinyint_e23 set tag tagname="abc" sql_error alter table st_tinyint_e24 set tag tagname=" " sql alter table st_tinyint_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_unsign.sim b/tests/script/tsim/parser/columnValue_unsign.sim index a72b1082f6..758814bc2b 100644 --- a/tests/script/tsim/parser/columnValue_unsign.sim +++ b/tests/script/tsim/parser/columnValue_unsign.sim @@ -130,3 +130,4 @@ if $rows != 1 then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index 8d0bc3fe5b..d50b53b2f7 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -237,8 +237,8 @@ if $rows != 3 then return -1 endi sql show databases -print wallevel $data13_testwal -if $data13_testwal != 1 then +print wallevel $data20_testwal +if $data20_testwal != 1 then return -1 endi sql drop database testwal diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim index 0913691a11..3a51186d55 100644 --- a/tests/script/tsim/valgrind/basic3.sim +++ b/tests/script/tsim/valgrind/basic3.sim @@ -50,11 +50,11 @@ while $i < $tbNum endw print =============== step3: tb -sql select tbcol5 - tbcol3 from tb1 -print =============== step4: stb -sql select tbcol5 - tbcol3 from stb +sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 1 soffset 1; +sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 2 soffset 4 limit 10 offset 1; +_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check $null= diff --git a/tests/script/tsim/valgrind/basic4.sim b/tests/script/tsim/valgrind/basic4.sim index 8be96f769b..98c4f8f2a8 100644 --- a/tests/script/tsim/valgrind/basic4.sim +++ b/tests/script/tsim/valgrind/basic4.sim @@ -22,41 +22,35 @@ if $data(1)[4] != ready then goto step1 endi -print =============== step2: create db -sql create database d1 vgroups 2 buffer 3 -sql show databases -sql use d1 -sql show vgroups +$tbPrefix = tb +$tbNum = 5 +$rowNum = 10 -print =============== step3: create show stable -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) -sql show stables -if $rows != 1 then - return -1 -endi +print =============== step2: prepare data +sql create database db vgroups 2 +sql use db +sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double, tbcol4 binary(30), tbcol5 binary(30)) tags (tgcol int unsigned) -print =============== step4: create show table -sql create table ct1 using stb tags(1000) -sql create table ct2 using stb tags(2000) -sql create table ct3 using stb tags(3000) -sql show tables -if $rows != 3 then - return -1 -endi +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , $x , $x , $x , "abcd1234=-+*" , "123456 0" ) + $x = $x + 1 + endw -print =============== step5: insert data (null / update) -sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, NULL)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql insert into ct2 values(now+0s, 10, 2.0, 3.0) -sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql insert into ct3 values('2021-01-01 00:00:00.000', NULL, NULL, 3.0) -sql insert into ct3 values('2022-03-02 16:59:00.010', 3 , 4, 5), ('2022-03-02 16:59:00.010', 33 , 4, 5), ('2022-04-01 16:59:00.011', 4, 4, 5), ('2022-04-01 16:59:00.011', 6, 4, 5), ('2022-03-06 16:59:00.013', 8, 4, 5); -sql insert into ct3 values('2022-03-02 16:59:00.010', 103, 1, 2), ('2022-03-02 16:59:00.010', 303, 3, 4), ('2022-04-01 16:59:00.011', 40, 5, 6), ('2022-04-01 16:59:00.011', 60, 4, 5), ('2022-03-06 16:59:00.013', 80, 4, 5); - -print =============== step6: query data= - -sql select * from stb where t1 between 1000 and 2500 + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , NULL , NULL , NULL , NULL , NULL ) + $i = $i + 1 +endw +print =============== step3: tb +sql select distinct(tbname), tgcol from stb; _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 6e456148bf..70f55370d2 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -111,6 +111,9 @@ sql select _wstart, count(*) from tb1 session(ts, 1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) sql select tbcol5 - tbcol3 from stb +sql select spread( tbcol2 )/44, spread(tbcol2), 0.204545455 * 44 from stb; +sql select min(tbcol) * max(tbcol) /4, sum(tbcol2) * apercentile(tbcol2, 20), apercentile(tbcol2, 33) + 52/9 from stb; + print =============== step5: explain sql explain analyze select ts from stb where -2; sql explain analyze select ts from tb1; From d0416f312f48d309a5f9299bdacc6e7cccd0cf6d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Aug 2022 19:07:07 +0800 Subject: [PATCH 61/62] fix: tdb use sys memory when snapshot writing --- source/dnode/vnode/src/inc/vnodeInt.h | 4 ++-- source/dnode/vnode/src/meta/metaCommit.c | 9 ++++++--- source/dnode/vnode/src/meta/metaSnapshot.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 47f7d209b3..2b154fce04 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -97,7 +97,7 @@ typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta); int metaClose(SMeta* pMeta); -int metaBegin(SMeta* pMeta); +int metaBegin(SMeta* pMeta, int8_t fromSys); int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); @@ -187,7 +187,7 @@ int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); int32_t smaDoRetention(SSma* pSma, int64_t now); -int32_t smaProcessFetch(SSma *pSma, void* pMsg); +int32_t smaProcessFetch(SSma* pSma, void* pMsg); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index b4987aea2b..85ed40970c 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -19,9 +19,12 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBuf static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn -int metaBegin(SMeta *pMeta) { - tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - +int metaBegin(SMeta *pMeta, int8_t fromSys) { + if (fromSys) { + tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } else { + tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) { return -1; } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index e01f0e7c01..973c381407 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -145,7 +145,7 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr pWriter->sver = sver; pWriter->ever = ever; - metaBegin(pMeta); + metaBegin(pMeta, 1); *ppWriter = pWriter; return code; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 460d0b6fbb..c8dc07af0a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -42,7 +42,7 @@ int vnodeBegin(SVnode *pVnode) { pVnode->state.commitID++; // begin meta - if (metaBegin(pVnode->pMeta) < 0) { + if (metaBegin(pVnode->pMeta, 0) < 0) { vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } From 42b5069586c207626498c4dfc4f5b2f87a903c96 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 19:41:16 +0800 Subject: [PATCH 62/62] test: valgrind case --- tests/script/tsim/valgrind/checkError8.sim | 152 +++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/script/tsim/valgrind/checkError8.sim diff --git a/tests/script/tsim/valgrind/checkError8.sim b/tests/script/tsim/valgrind/checkError8.sim new file mode 100644 index 0000000000..7ca01bc3d0 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError8.sim @@ -0,0 +1,152 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 5 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + + +print ============== query +sql select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and ((a.c1 > 50 and a.c1 < 60) or (b.c2 > 60));; +sql select * from stb1 where (c6 > 3.0 or c6 < 60) and c6 > 50 and (c6 != 53 or c6 != 63);; +sql select ts,c1 from stb1 where (c1 > 60 or c1 < 10 or (c1 > 20 and c1 < 30)) and ts > '2021-05-05 18:19:00.000' and ts < '2021-05-05 18:19:25.000' and c1 != 21 and c1 != 22 order by ts; +sql select a.* from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50 order by ts;; +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and (a.c1 < 10 or a.c1 > 30) and (b.u1 < 5 or b.u1 > 5) order by ts;; +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and a.c1 < 30 and b.u1 > 1 and a.c1 > 10 and b.u1 < 8 and b.u1<>5 order by ts;; +sql select tb1.ts,tb1.*,tb2_1.* from tb1, tb2_1 where tb1.ts=tb2_1.ts and tb1.ts >= '2021-05-05 18:19:03.000' and tb1.c7=false and tb2_1.u3>4 order by ts;; +sql select stb1.ts,stb1.c1,stb1.t1,stb2.ts,stb2.u1,stb2.t4 from stb1, stb2 where stb1.ts=stb2.ts and stb1.t1 = stb2.t4 order by ts;; +sql select count(*) from stb1 where tbname like 'tb%' or c1 > 0;; +sql select * from stb1 where tbname like 'tb%' and (t1=1 or t2=2 or t3=3) and t1 > 2 order by ts;; + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 2 then + return -1 +endi + +if $system_content == $null then + return -1 +endi