xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/3_liaohj

This commit is contained in:
Haojun Liao 2022-06-20 14:08:21 +08:00
parent 868185a3b9 3155a228e5
commit ad0595b9ec
100 changed files with 4602 additions and 2737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/14-reference/06-taosdump.md
View File

@ -36,6 +36,8 @@ There are two ways to install taosdump:
:::tip
- taosdump versions after 1.4.1 provide the `-I` argument for parsing Avro file schema and data. If users specify `-s` then only taosdump will parse schema.
- Backups after taosdump 1.4.2 use the batch count specified by the `-B` parameter. The default value is 16384. If, in some environments, low network speed or disk performance causes "Error actual dump ... batch ...", then try changing the `-B` parameter to a smaller value.
- The export of taosdump does not support resuming from an interruption. Therefore, if the taosdump process terminates unexpectedly, delete all related files that have been exported or generated.
- The import of taosdump supports resuming from an interruption, but when the process resumes, you will receive some "table already exists" messages, which could be ignored.
:::

2
docs/zh/14-reference/06-taosdump.md
View File

@ -39,6 +39,8 @@ taosdump 有两种安装方式:
:::tip
- taosdump 1.4.1 之后的版本提供 `-I` 参数，用于解析 avro 文件 schema 和数据，如果指定 `-s` 参数将只解析 schema。
- taosdump 1.4.2 之后的备份使用 `-B` 参数指定的批次数，默认值为 16384，如果在某些环境下由于网络速度或磁盘性能不足导致 "Error actual dump .. batch .." 可以通过 `-B` 参数调整为更小的值进行尝试。
- taosdump 的导出不支持中断恢复，所以当进程意外终止后，正确的处理方式是删除当前已导出或生成的所有相关文件。
- taosdump 的导入支持中断恢复，但是当进程重新启动时，会收到一些“表已经存在”的提示，可以忽视。
:::

5
include/common/tmsg.h
View File

@ -1817,6 +1817,8 @@ typedef struct SVCreateTbReq {
tb_uid_t uid;
int64_t ctime;
int32_t ttl;
int32_t commentLen;
char* comment;
int8_t type;
union {
struct {
@ -1834,6 +1836,7 @@ int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq);
static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
taosMemoryFreeClear(req->name);
taosMemoryFreeClear(req->comment);
if (req->type == TSDB_CHILD_TABLE) {
taosMemoryFreeClear(req->ctb.pTag);
} else if (req->type == TSDB_NORMAL_TABLE) {
@ -1930,7 +1933,7 @@ typedef struct {
// TSDB_ALTER_TABLE_UPDATE_OPTIONS
int8_t updateTTL;
int32_t newTTL;
int8_t updateComment;
int32_t newCommentLen;
char* newComment;
} SVAlterTbReq;

2
include/common/tmsgdef.h
View File

@ -198,7 +198,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_QND_MSG)
//shared by snode and vnode

30
include/common/ttokendef.h
View File

@ -241,20 +241,22 @@
#define TK_LINEAR 223
#define TK_NEXT 224
#define TK_HAVING 225
#define TK_ORDER 226
#define TK_SLIMIT 227
#define TK_SOFFSET 228
#define TK_LIMIT 229
#define TK_OFFSET 230
#define TK_ASC 231
#define TK_NULLS 232
#define TK_ID 233
#define TK_NK_BITNOT 234
#define TK_INSERT 235
#define TK_VALUES 236
#define TK_IMPORT 237
#define TK_NK_SEMI 238
#define TK_FILE 239
#define TK_RANGE 226
#define TK_EVERY 227
#define TK_ORDER 228
#define TK_SLIMIT 229
#define TK_SOFFSET 230
#define TK_LIMIT 231
#define TK_OFFSET 232
#define TK_ASC 233
#define TK_NULLS 234
#define TK_ID 235
#define TK_NK_BITNOT 236
#define TK_INSERT 237
#define TK_VALUES 238
#define TK_IMPORT 239
#define TK_NK_SEMI 240
#define TK_FILE 241
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

1
include/dnode/mnode/mnode.h
View File

@ -96,6 +96,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
int32_t mndProcessRpcMsg(SRpcMsg *pMsg);
int32_t mndProcessSyncMsg(SRpcMsg *pMsg);
int32_t mndPreProcessMsg(SRpcMsg *pMsg);
void mndAbortPreprocessMsg(SRpcMsg *pMsg);
/**
* @brief Generate machine code

2
include/libs/catalog/catalog.h
View File

@ -87,7 +87,7 @@ typedef struct SMetaData {
SArray* pUdfList; // pRes = SFuncInfo*
SArray* pIndex; // pRes = SIndexInfo*
SArray* pUser; // pRes = bool*
SArray* pQnodeList; // pRes = SQueryNodeAddr*
SArray* pQnodeList; // pRes = SArray<SQueryNodeLoad>*
} SMetaData;
typedef struct SCatalogCfg {

1
include/libs/function/functionMgt.h
View File

@ -189,6 +189,7 @@ bool fmIsForbidStreamFunc(int32_t funcId);
bool fmIsForbidWindowFunc(int32_t funcId);
bool fmIsForbidGroupByFunc(int32_t funcId);
bool fmIsIntervalInterpoFunc(int32_t funcId);
bool fmIsInterpFunc(int32_t funcId);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);

2
include/libs/nodes/cmdnodes.h
View File

@ -87,6 +87,7 @@ typedef struct SAlterDatabaseStmt {
typedef struct STableOptions {
ENodeType type;
bool commentNull;
char comment[TSDB_TB_COMMENT_LEN];
SNodeList* pMaxDelay;
int64_t maxDelay1;
@ -126,6 +127,7 @@ typedef struct SCreateSubTableClause {
bool ignoreExists;
SNodeList* pSpecificTags;
SNodeList* pValsOfTags;
STableOptions* pOptions;
} SCreateSubTableClause;
typedef struct SCreateMultiTableStmt {

2
include/libs/nodes/nodes.h
View File

@ -205,6 +205,7 @@ typedef enum ENodeType {
QUERY_NODE_LOGIC_PLAN_SORT,
QUERY_NODE_LOGIC_PLAN_PARTITION,
QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_LOGIC_PLAN_INTERP_FUNC,
QUERY_NODE_LOGIC_SUBPLAN,
QUERY_NODE_LOGIC_PLAN,
@ -236,6 +237,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE,
QUERY_NODE_PHYSICAL_PLAN_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC,
QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
QUERY_NODE_PHYSICAL_PLAN_INSERT,
QUERY_NODE_PHYSICAL_PLAN_DELETE,

29
include/libs/nodes/plannodes.h
View File

@ -98,9 +98,19 @@ typedef struct SProjectLogicNode {
typedef struct SIndefRowsFuncLogicNode {
SLogicNode node;
SNodeList* pVectorFuncs;
SNodeList* pFuncs;
} SIndefRowsFuncLogicNode;
typedef struct SInterpFuncLogicNode {
SLogicNode node;
SNodeList* pFuncs;
STimeWindow timeRange;
int64_t interval;
EFillMode fillMode;
SNode* pFillValues; // SNodeListNode
SNode* pTimeSeries; // SColumnNode
} SInterpFuncLogicNode;
typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType;
typedef struct SVnodeModifyLogicNode {
@ -154,6 +164,7 @@ typedef struct SWindowLogicNode {
int8_t slidingUnit;
int64_t sessionGap;
SNode* pTspk;
SNode* pTsEnd;
SNode* pStateExpr;
int8_t triggerType;
int64_t watermark;
@ -292,9 +303,20 @@ typedef struct SProjectPhysiNode {
typedef struct SIndefRowsFuncPhysiNode {
SPhysiNode node;
SNodeList* pExprs;
SNodeList* pVectorFuncs;
SNodeList* pFuncs;
} SIndefRowsFuncPhysiNode;
typedef struct SInterpFuncPhysiNode {
SPhysiNode node;
SNodeList* pExprs;
SNodeList* pFuncs;
STimeWindow timeRange;
int64_t interval;
EFillMode fillMode;
SNode* pFillValues; // SNodeListNode
SNode* pTimeSeries; // SColumnNode
} SInterpFuncPhysiNode;
typedef struct SJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
@ -337,7 +359,8 @@ typedef struct SWinodwPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of parameter expression of function
SNodeList* pFuncs;
SNode* pTspk; // timestamp primary key
SNode* pTspk; // timestamp primary key
SNode* pTsEnd; // window end timestamp
int8_t triggerType;
int64_t watermark;
double filesFactor;

4
include/libs/nodes/querynodes.h
View File

@ -240,6 +240,9 @@ typedef struct SSelectStmt {
SNode* pWindow;
SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving;
SNode* pRange;
SNode* pEvery;
SNode* pFill;
SNodeList* pOrderByList; // SOrderByExprNode
SLimitNode* pLimit;
SLimitNode* pSlimit;
@ -254,6 +257,7 @@ typedef struct SSelectStmt {
bool hasSelectValFunc;
bool hasUniqueFunc;
bool hasTailFunc;
bool hasInterpFunc;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;

4
include/libs/qcom/query.h
View File

@ -47,6 +47,10 @@ typedef enum {
TARGET_TYPE_OTHER,
} ETargetType;
#define QUERY_POLICY_VNODE 1
#define QUERY_POLICY_HYBRID 2
#define QUERY_POLICY_QNODE 3
typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema
uint8_t precision; // the number of precision

2
include/libs/qworker/qworker.h
View File

@ -64,6 +64,8 @@ typedef struct {
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb);
int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);

7
include/libs/sync/sync.h
View File

@ -107,7 +107,9 @@ typedef struct SSyncFSM {
void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta);
void (*FpLeaderTransferCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void *pReaderParam, void** ppReader);
int32_t (*FpGetSnapshotInfo)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
int32_t (*FpSnapshotStartRead)(struct SSyncFSM* pFsm, void** ppReader);
int32_t (*FpSnapshotStopRead)(struct SSyncFSM* pFsm, void* pReader);
@ -193,8 +195,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
bool syncEnvIsStart();
const char* syncStr(ESyncState state);
bool syncIsRestoreFinish(int64_t rid);
int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta);
int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct SSnapshotMeta* sMeta);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg);

2
include/util/taoserror.h
View File

@ -128,6 +128,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X0226)
#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0227)
#define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X0228)
#define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X0229)
// mnode-common
#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300)
@ -563,6 +564,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x265A)
#define TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x265B)
#define TSDB_CODE_PAR_INVALID_TABLE_OPTION TAOS_DEF_ERROR_CODE(0, 0x265C)
#define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x265D)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)

8
source/client/inc/clientInt.h
View File

@ -22,6 +22,7 @@ extern "C" {
#include "parser.h"
#include "planner.h"
#include "catalog.h"
#include "query.h"
#include "taos.h"
#include "tcommon.h"
@ -162,6 +163,7 @@ typedef struct SReqResultInfo {
int32_t precision;
bool convertUcs4;
int32_t payloadLen;
char* convertJson;
} SReqResultInfo;
typedef struct SRequestSendRecvBody {
@ -242,6 +244,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver
taosMemoryFreeClear(msg->resInfo.pCol);
taosMemoryFreeClear(msg->resInfo.length);
taosMemoryFreeClear(msg->resInfo.convertBuf);
taosMemoryFreeClear(msg->resInfo.convertJson);
}
setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false);
return &msg->resInfo;
@ -292,7 +295,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen);
int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray** pNodeList);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
@ -317,12 +320,13 @@ void hbMgrInitMqHbRspHandle();
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res);
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery);
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta);
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
int32_t removeMeta(STscObj* pTscObj, SArray* tbList);// todo move to clientImpl.c and become a static function
int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);// todo move to xxx
bool qnodeRequired(SRequestObj* pRequest);
#ifdef __cplusplus
}

1
source/client/src/clientEnv.c
View File

@ -212,6 +212,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
taosMemoryFreeClear(pResInfo->pCol);
taosMemoryFreeClear(pResInfo->fields);
taosMemoryFreeClear(pResInfo->userFields);
taosMemoryFreeClear(pResInfo->convertJson);
if (pResInfo->convertBuf != NULL) {
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {

468
source/client/src/clientImpl.c
View File

@ -307,6 +307,21 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
return TSDB_CODE_SUCCESS;
}
bool qnodeRequired(SRequestObj* pRequest) {
if (QUERY_POLICY_VNODE == tsQueryPolicy) {
return false;
}
SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo;
bool required = false;
taosThreadMutexLock(&pInfo->qnodeMutex);
required = (NULL == pInfo->pQnodeList);
taosThreadMutexUnlock(&pInfo->qnodeMutex);
return required;
}
int32_t getQnodeList(SRequestObj* pRequest, SArray** pNodeList) {
SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo;
int32_t code = 0;
@ -337,7 +352,7 @@ int32_t getQnodeList(SRequestObj* pRequest, SArray** pNodeList) {
return code;
}
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray** pNodeList) {
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
pRequest->type = pQuery->msgType;
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
@ -349,12 +364,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
int32_t code = getQnodeList(pRequest, pNodeList);
if (TSDB_CODE_SUCCESS == code) {
code = qCreateQueryPlan(&cxt, pPlan, *pNodeList);
}
return code;
return qCreateQueryPlan(&cxt, pPlan, pNodeList);
}
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) {
@ -397,6 +407,195 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) {
pResInfo->precision = precision;
}
int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pDbVgList) {
SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
int32_t dbNum = taosArrayGetSize(pDbVgList);
for (int32_t i = 0; i < dbNum; ++i) {
SArray* pVg = taosArrayGetP(pDbVgList, i);
int32_t vgNum = taosArrayGetSize(pVg);
if (vgNum <= 0) {
continue;
}
for (int32_t j = 0; j < vgNum; ++j) {
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
SQueryNodeLoad load = {0};
load.addr.nodeId = pInfo->vgId;
load.addr.epSet = pInfo->epSet;
taosArrayPush(nodeList, &load);
}
}
int32_t vnodeNum = taosArrayGetSize(nodeList);
if (vnodeNum > 0) {
tscDebug("0x%" PRIx64 " vnode policy, use vnode list, num:%d", pRequest->requestId, vnodeNum);
goto _return;
}
int32_t mnodeNum = taosArrayGetSize(pMnodeList);
if (mnodeNum <= 0) {
tscDebug("0x%" PRIx64 " vnode policy, empty node list", pRequest->requestId);
goto _return;
}
void* pData = taosArrayGet(pMnodeList, 0);
taosArrayAddBatch(nodeList, pData, mnodeNum);
tscDebug("0x%" PRIx64 " vnode policy, use mnode list, num:%d", pRequest->requestId, mnodeNum);
_return:
*pNodeList = nodeList;
return TSDB_CODE_SUCCESS;
}
int32_t buildQnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pQnodeList) {
SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
int32_t qNodeNum = taosArrayGetSize(pQnodeList);
if (qNodeNum > 0) {
void* pData = taosArrayGet(pQnodeList, 0);
taosArrayAddBatch(nodeList, pData, qNodeNum);
tscDebug("0x%" PRIx64 " qnode policy, use qnode list, num:%d", pRequest->requestId, qNodeNum);
goto _return;
}
int32_t mnodeNum = taosArrayGetSize(pMnodeList);
if (mnodeNum <= 0) {
tscDebug("0x%" PRIx64 " qnode policy, empty node list", pRequest->requestId);
goto _return;
}
void* pData = taosArrayGet(pMnodeList, 0);
taosArrayAddBatch(nodeList, pData, mnodeNum);
tscDebug("0x%" PRIx64 " qnode policy, use mnode list, num:%d", pRequest->requestId, mnodeNum);
_return:
*pNodeList = nodeList;
return TSDB_CODE_SUCCESS;
}
int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) {
SArray* pDbVgList = NULL;
SArray* pQnodeList = NULL;
int32_t code = 0;
switch (tsQueryPolicy) {
case QUERY_POLICY_VNODE: {
if (pResultMeta) {
pDbVgList = taosArrayInit(4, POINTER_BYTES);
int32_t dbNum = taosArrayGetSize(pResultMeta->pDbVgroup);
for (int32_t i = 0; i < dbNum; ++i) {
SMetaRes* pRes = taosArrayGet(pResultMeta->pDbVgroup, i);
if (pRes->code || NULL == pRes->pRes) {
continue;
}
taosArrayPush(pDbVgList, &pRes->pRes);
}
}
code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList);
break;
}
case QUERY_POLICY_HYBRID:
case QUERY_POLICY_QNODE: {
if (pResultMeta && taosArrayGetSize(pResultMeta->pQnodeList) > 0) {
SMetaRes* pRes = taosArrayGet(pResultMeta->pQnodeList, 0);
if (pRes->code) {
pQnodeList = NULL;
} else {
pQnodeList = taosArrayDup((SArray*)pRes->pRes);
}
} else {
SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo;
taosThreadMutexLock(&pInst->qnodeMutex);
if (pInst->pQnodeList) {
pQnodeList = taosArrayDup(pInst->pQnodeList);
}
taosThreadMutexUnlock(&pInst->qnodeMutex);
}
code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList);
break;
}
default:
tscError("unknown query policy: %d", tsQueryPolicy);
return TSDB_CODE_TSC_APP_ERROR;
}
taosArrayDestroy(pDbVgList);
taosArrayDestroy(pQnodeList);
return code;
}
int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList) {
SArray* pDbVgList = NULL;
SArray* pQnodeList = NULL;
int32_t code = 0;
switch (tsQueryPolicy) {
case QUERY_POLICY_VNODE: {
int32_t dbNum = taosArrayGetSize(pRequest->dbList);
if (dbNum > 0) {
SCatalog* pCtg = NULL;
SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo;
code = catalogGetHandle(pInst->clusterId, &pCtg);
if (code != TSDB_CODE_SUCCESS) {
goto _return;
}
pDbVgList = taosArrayInit(dbNum, POINTER_BYTES);
SArray* pVgList = NULL;
for (int32_t i = 0; i < dbNum; ++i) {
char* dbFName = taosArrayGet(pRequest->dbList, i);
SRequestConnInfo conn = {.pTrans = pInst->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pInst->mgmtEp)};
code = catalogGetDBVgInfo(pCtg, &conn, dbFName, &pVgList);
if (code) {
goto _return;
}
taosArrayPush(pDbVgList, &pVgList);
}
}
code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList);
break;
}
case QUERY_POLICY_HYBRID:
case QUERY_POLICY_QNODE: {
getQnodeList(pRequest, &pQnodeList);
code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList);
break;
}
default:
tscError("unknown query policy: %d", tsQueryPolicy);
return TSDB_CODE_TSC_APP_ERROR;
}
_return:
taosArrayDestroy(pDbVgList);
taosArrayDestroy(pQnodeList);
return code;
}
int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
tsem_init(&schdRspSem, 0, 0);
@ -658,12 +857,16 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
code = execDdlQuery(pRequest, pQuery);
break;
case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pNodeList = NULL;
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, &pNodeList);
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pMnodeList);
if (TSDB_CODE_SUCCESS == code) {
SArray* pNodeList = NULL;
buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList);
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
taosArrayDestroy(pNodeList);
}
taosArrayDestroy(pNodeList);
taosArrayDestroy(pMnodeList);
break;
}
case QUERY_EXEC_MODE_EMPTY_RESULT:
@ -712,7 +915,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
return launchQueryImpl(pRequest, pQuery, false, NULL);
}
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) {
int32_t code = 0;
switch (pQuery->execMode) {
@ -723,7 +926,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
code = asyncExecDdlQuery(pRequest, pQuery);
break;
case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pNodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
pRequest->type = pQuery->msgType;
@ -736,13 +939,16 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pNodeList);
code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList);
if (code) {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
}
if (TSDB_CODE_SUCCESS == code) {
SArray* pNodeList = NULL;
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
SRequestConnInfo conn = {
.pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
SSchedulerReq req = {.pConn = &conn,
@ -754,6 +960,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
.cbParam = pRequest,
.reqKilled = &pRequest->killed};
code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
taosArrayDestroy(pNodeList);
} else {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
@ -761,7 +968,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
}
// todo not to be released here
taosArrayDestroy(pNodeList);
taosArrayDestroy(pMnodeList);
break;
}
case QUERY_EXEC_MODE_EMPTY_RESULT:
@ -1339,67 +1546,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
} else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
int32_t len = 0;
SResultColumn* pCol = &pResultInfo->pCol[i];
for (int32_t j = 0; j < numOfRows; ++j) {
if (pCol->offset[j] != -1) {
char* pStart = pCol->offset[j] + pCol->pData;
int32_t jsonInnerType = *pStart;
char* jsonInnerData = pStart + CHAR_BYTES;
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (jsonInnerType == TD_TAG_JSON) {
char* jsonString = parseTagDatatoJson(pStart);
STR_TO_VARSTR(dst, jsonString);
taosMemoryFree(jsonString);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
*(char*)varDataVal(dst) = '\"';
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(dst) + CHAR_BYTES);
if (length <= 0) {
tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
length = 0;
}
varDataSetLen(dst, length + CHAR_BYTES * 2);
*(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"';
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
double jsonVd = *(double*)(jsonInnerData);
sprintf(varDataVal(dst), "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
} else {
ASSERT(0);
}
if (len + varDataTLen(dst) > colLength[i]) {
p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
}
p = pResultInfo->convertBuf[i] + len;
memcpy(p, dst, varDataTLen(dst));
pCol->offset[j] = len;
len += varDataTLen(dst);
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
@ -1408,6 +1554,171 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
return TSDB_CODE_SUCCESS;
}
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){
char* p = (char*)pResultInfo->pData;
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
int32_t* colLength = (int32_t*)(p + len);
len += sizeof(int32_t) * numOfCols;
char* pStart = p + len;
for (int32_t i = 0; i < numOfCols; ++i) {
int32_t colLen = htonl(colLength[i]);
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
int32_t* offset = (int32_t*)pStart;
int32_t lenTmp = numOfRows * sizeof(int32_t);
len += lenTmp;
pStart += lenTmp;
for (int32_t j = 0; j < numOfRows; ++j) {
if (offset[j] == -1) {
continue;
}
char* data = offset[j] + pStart;
int32_t jsonInnerType = *data;
char* jsonInnerData = data + CHAR_BYTES;
if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
len += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L));
} else if (jsonInnerType & TD_TAG_JSON) {
len += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
len += varDataTLen(jsonInnerData) + CHAR_BYTES * 2;
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
len += (VARSTR_HEADER_SIZE + 32);
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
len += (VARSTR_HEADER_SIZE + 5);
} else {
ASSERT(0);
}
}
} else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
int32_t lenTmp = numOfRows * sizeof(int32_t);
len += (lenTmp + colLen);
pStart += lenTmp;
} else {
int32_t lenTmp = BitmapLen(pResultInfo->numOfRows);
len += (lenTmp + colLen);
pStart += lenTmp;
}
pStart += colLen;
}
return len;
}
static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
bool needConvert = false;
for (int32_t i = 0; i < numOfCols; ++i) {
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
needConvert = true;
break;
}
}
if(!needConvert) return TSDB_CODE_SUCCESS;
char* p = (char*)pResultInfo->pData;
int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows);
pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char* p1 = pResultInfo->convertJson;
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
memcpy(p1, p, len);
p += len;
p1 += len;
len = sizeof(int32_t) * numOfCols;
int32_t* colLength = (int32_t*)p;
int32_t* colLength1 = (int32_t*)p1;
memcpy(p1, p, len);
p += len;
p1 += len;
char* pStart = p;
char* pStart1 = p1;
for (int32_t i = 0; i < numOfCols; ++i) {
int32_t colLen = htonl(colLength[i]);
int32_t colLen1 = htonl(colLength1[i]);
ASSERT(colLen < dataLen);
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
int32_t* offset = (int32_t*)pStart;
int32_t* offset1 = (int32_t*)pStart1;
len = numOfRows * sizeof(int32_t);
memcpy(pStart1, pStart, len);
pStart += len;
pStart1 += len;
len = 0;
for (int32_t j = 0; j < numOfRows; ++j) {
if (offset[j] == -1) {
continue;
}
char* data = offset[j] + pStart;
int32_t jsonInnerType = *data;
char* jsonInnerData = data + CHAR_BYTES;
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (jsonInnerType & TD_TAG_JSON) {
char* jsonString = parseTagDatatoJson(data);
STR_TO_VARSTR(dst, jsonString);
taosMemoryFree(jsonString);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
*(char*)varDataVal(dst) = '\"';
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(dst) + CHAR_BYTES);
if (length <= 0) {
tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
length = 0;
}
varDataSetLen(dst, length + CHAR_BYTES * 2);
*(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"';
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
double jsonVd = *(double*)(jsonInnerData);
sprintf(varDataVal(dst), "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
} else {
ASSERT(0);
}
offset1[j]= len;
memcpy(pStart1 + len, dst, varDataTLen(dst));
len += varDataTLen(dst);
}
colLen1 = len;
colLength1[i] = htonl(len);
} else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
len = numOfRows * sizeof(int32_t);
memcpy(pStart1, pStart, len);
pStart += len;
pStart1 += len;
memcpy(pStart1, pStart, colLen);
} else {
len = BitmapLen(pResultInfo->numOfRows);
memcpy(pStart1, pStart, len);
pStart += len;
pStart1 += len;
memcpy(pStart1, pStart, colLen);
}
pStart += colLen;
pStart1 += colLen1;
}
pResultInfo->pData = pResultInfo->convertJson;
return TSDB_CODE_SUCCESS;
}
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
bool convertUcs4) {
assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
@ -1419,6 +1730,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = doConvertJson(pResultInfo, numOfCols, numOfRows);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
char* p = (char*)pResultInfo->pData;
@ -1462,8 +1777,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
pStart += colLength[i];
}
// convert UCS4-LE encoded character to native multi-bytes character in current data block.
if (convertUcs4) {
if(convertUcs4){
code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength);
}

4
source/client/src/clientMain.c
View File

@ -702,7 +702,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
TSWAP(pRequest->tableList, (pQuery)->pTableList);
destorySqlParseWrapper(pWrapper);
launchAsyncQuery(pRequest, pQuery);
launchAsyncQuery(pRequest, pQuery, pResultMeta);
} else {
destorySqlParseWrapper(pWrapper);
tscDebug("error happens, code:%d", code);
@ -808,7 +808,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
SQuery *pQuery = NULL;
SCatalogReq catalogReq = {.forceUpdate = updateMetaForce};
SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)};
code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq);
if (code != TSDB_CODE_SUCCESS) {
goto _error;

2
source/client/src/clientSml.c
View File

@ -2274,7 +2274,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
// info->affectedRows = taos_affected_rows(info->pRequest);
// return info->pRequest->code;
launchAsyncQuery(info->pRequest, info->pQuery);
launchAsyncQuery(info->pRequest, info->pQuery, NULL);
return TSDB_CODE_SUCCESS;
}

4
source/common/src/systable.c
View File

@ -124,7 +124,7 @@ static const SSysDbTableSchema userStbsSchema[] = {
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
};
static const SSysDbTableSchema streamSchema[] = {
@ -148,7 +148,7 @@ static const SSysDbTableSchema userTblsSchema[] = {
{.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
};

24
source/common/src/tmsg.c
View File

@ -524,7 +524,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
}
if (pReq->commentLen > 0) {
if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1;
}
if (pReq->ast1Len > 0) {
if (tEncodeBinary(&encoder, pReq->pAst1, pReq->ast1Len) < 0) return -1;
@ -589,7 +589,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
}
if (pReq->commentLen > 0) {
pReq->comment = taosMemoryMalloc(pReq->commentLen);
pReq->comment = taosMemoryMalloc(pReq->commentLen + 1);
if (pReq->comment == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
}
@ -707,7 +707,7 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
if (pReq->commentLen > 0) {
pReq->comment = taosMemoryMalloc(pReq->commentLen);
pReq->comment = taosMemoryMalloc(pReq->commentLen + 1);
if (pReq->comment == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
}
@ -4374,6 +4374,10 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1;
if (tEncodeI8(pCoder, pReq->type) < 0) return -1;
if (tEncodeI32(pCoder, pReq->commentLen) < 0) return -1;
if (pReq->commentLen > 0) {
if (tEncodeCStr(pCoder, pReq->comment) < 0) return -1;
}
if (pReq->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1;
@ -4397,6 +4401,12 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1;
if (tDecodeI8(pCoder, &pReq->type) < 0) return -1;
if (tDecodeI32(pCoder, &pReq->commentLen) < 0) return -1;
if (pReq->commentLen > 0) {
pReq->comment = taosMemoryMalloc(pReq->commentLen + 1);
if (pReq->comment == NULL) return -1;
if (tDecodeCStrTo(pCoder, pReq->comment) < 0) return -1;
}
if (pReq->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1;
@ -4750,8 +4760,8 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
if (pReq->updateTTL) {
if (tEncodeI32v(pEncoder, pReq->newTTL) < 0) return -1;
}
if (tEncodeI8(pEncoder, pReq->updateComment) < 0) return -1;
if (pReq->updateComment) {
if (tEncodeI32v(pEncoder, pReq->newCommentLen) < 0) return -1;
if (pReq->newCommentLen > 0) {
if (tEncodeCStr(pEncoder, pReq->newComment) < 0) return -1;
}
break;
@ -4798,8 +4808,8 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
if (pReq->updateTTL) {
if (tDecodeI32v(pDecoder, &pReq->newTTL) < 0) return -1;
}
if (tDecodeI8(pDecoder, &pReq->updateComment) < 0) return -1;
if (pReq->updateComment) {
if (tDecodeI32v(pDecoder, &pReq->newCommentLen) < 0) return -1;
if (pReq->newCommentLen > 0) {
if (tDecodeCStr(pDecoder, &pReq->newComment) < 0) return -1;
}
break;

1
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -334,6 +334,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

40
source/dnode/mnode/impl/src/mndMain.c
View File

@ -77,12 +77,50 @@ static void mndPullupTelem(SMnode *pMnode) {
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
}
static void mndPushTtlTime(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
mError("ttl time malloc err. contLen:%d", contLen);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
int32_t t = taosGetTimestampSec();
*(int32_t*)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if(code != 0){
mError("ttl time seed err. code:%d", code);
}
mError("ttl time seed succ. time:%d", t);
sdbRelease(pSdb, pVgroup);
}
}
static void *mndThreadFp(void *param) {
SMnode *pMnode = param;
int64_t lastTime = 0;
setThreadName("mnode-timer");
while (1) {
if (lastTime % (864000) == 0) { // sleep 1 day for ttl
mndPushTtlTime(pMnode);
}
lastTime++;
taosMsleep(100);
if (mndGetStop(pMnode)) break;
@ -512,6 +550,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
pMsg->msgType != TDMT_MND_TRANS_TIMER) {
mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
mndAbortPreprocessMsg(pMsg);
SEpSet epSet = {0};
mndGetMnodeEpSet(pMsg->info.node, &epSet);

7
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -25,6 +25,13 @@ int32_t mndPreProcessMsg(SRpcMsg *pMsg) {
return qWorkerPreprocessQueryMsg(pMnode->pQuery, pMsg);
}
void mndAbortPreprocessMsg(SRpcMsg *pMsg) {
if (TDMT_VND_QUERY != pMsg->msgType) return;
SMnode *pMnode = pMsg->info.node;
qWorkerAbortPreprocessQueryMsg(pMnode->pQuery, pMsg);
}
int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
int32_t code = -1;
SMnode *pMnode = pMsg->info.node;

51
source/dnode/mnode/impl/src/mndStb.c
View File

@ -117,7 +117,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
}
if (pStb->commentLen > 0) {
SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen + 1, _OVER)
}
if (pStb->ast1Len > 0) {
SDB_SET_BINARY(pRaw, dataPos, pStb->pAst1, pStb->ast1Len, _OVER)
@ -204,9 +204,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
}
if (pStb->commentLen > 0) {
pStb->comment = taosMemoryCalloc(pStb->commentLen, 1);
pStb->comment = taosMemoryCalloc(pStb->commentLen + 1, 1);
if (pStb->comment == NULL) goto _OVER;
SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen + 1, _OVER)
}
if (pStb->ast1Len > 0) {
pStb->pAst1 = taosMemoryCalloc(pStb->ast1Len, 1);
@ -280,8 +280,8 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
}
}
if (pOld->commentLen < pNew->commentLen) {
void *comment = taosMemoryMalloc(pNew->commentLen);
if (pOld->commentLen < pNew->commentLen && pNew->commentLen > 0) {
void *comment = taosMemoryMalloc(pNew->commentLen + 1);
if (comment != NULL) {
taosMemoryFree(pOld->comment);
pOld->comment = comment;
@ -291,6 +291,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
taosWUnLockLatch(&pOld->lock);
}
}
pOld->commentLen = pNew->commentLen;
if (pOld->ast1Len < pNew->ast1Len) {
void *pAst1 = taosMemoryMalloc(pNew->ast1Len);
@ -330,8 +331,8 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
pOld->numOfTags = pNew->numOfTags;
memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema));
}
if (pNew->commentLen != 0) {
memcpy(pOld->comment, pNew->comment, pNew->commentLen);
if (pNew->commentLen > 0) {
memcpy(pOld->comment, pNew->comment, pNew->commentLen + 1);
}
if (pNew->ast1Len != 0) {
memcpy(pOld->pAst1, pNew->pAst1, pNew->ast1Len);
@ -676,12 +677,12 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pDst->numOfTags = pCreate->numOfTags;
pDst->commentLen = pCreate->commentLen;
if (pDst->commentLen > 0) {
pDst->comment = taosMemoryCalloc(pDst->commentLen, 1);
pDst->comment = taosMemoryCalloc(pDst->commentLen + 1, 1);
if (pDst->comment == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
memcpy(pDst->comment, pCreate->comment, pDst->commentLen);
memcpy(pDst->comment, pCreate->comment, pDst->commentLen + 1);
}
pDst->ast1Len = pCreate->ast1Len;
@ -835,7 +836,7 @@ _OVER:
}
static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
if (pAlter->commentLen != 0 || pAlter->ttl != 0) return 0;
if (pAlter->commentLen >= 0 || pAlter->ttl != 0) return 0;
if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
@ -890,13 +891,16 @@ static int32_t mndUpdateStbCommentAndTTL(const SStbObj *pOld, SStbObj *pNew, cha
int32_t ttl) {
if (commentLen > 0) {
pNew->commentLen = commentLen;
pNew->comment = taosMemoryCalloc(1, commentLen);
pNew->comment = taosMemoryCalloc(1, commentLen + 1);
if (pNew->comment == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
memcpy(pNew->comment, pComment, commentLen);
memcpy(pNew->comment, pComment, commentLen + 1);
} else if(commentLen == 0){
pNew->commentLen = 0;
}
if (ttl >= 0) {
pNew->ttl = ttl;
}
@ -1206,7 +1210,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
sdbRelease(pSdb, pVgroup);
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pReq;
@ -1841,17 +1844,17 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables
char *p = taosMemoryCalloc(1, pStb->commentLen + 1 + VARSTR_HEADER_SIZE); // check malloc failures
if (p != NULL) {
if (pStb->commentLen != 0) {
STR_TO_VARSTR(p, pStb->comment);
} else {
STR_TO_VARSTR(p, "");
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
colDataAppend(pColInfo, numOfRows, (const char *)p, false);
taosMemoryFree(p);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
if (pStb->commentLen > 0) {
char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(comment, pStb->comment);
colDataAppend(pColInfo, numOfRows, comment, false);
} else if(pStb->commentLen == 0) {
char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(comment, "");
colDataAppend(pColInfo, numOfRows, comment, false);
} else {
colDataAppendNULL(pColInfo, numOfRows);
}
numOfRows++;

13
source/dnode/mnode/impl/src/mndSync.c
View File

@ -73,7 +73,17 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
}
}
int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) {
// TODO:
// atomic operation
// step1. sdbGetCommitInfo
// step2. create ppReader with pReaderParam
return 0;
}
int32_t mndSyncGetSnapshotInfo(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
SMnode *pMnode = pFsm->data;
sdbGetCommitInfo(pMnode->pSdb, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, &pSnapshot->lastConfigIndex);
return 0;
@ -159,6 +169,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {
pFsm->FpRestoreFinishCb = mndRestoreFinish;
pFsm->FpReConfigCb = mndReConfig;
pFsm->FpGetSnapshot = mndSyncGetSnapshot;
pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo;
pFsm->FpSnapshotStartRead = mndSnapshotStartRead;
pFsm->FpSnapshotStopRead = mndSnapshotStopRead;
pFsm->FpSnapshotDoRead = mndSnapshotDoRead;

4
source/dnode/vnode/inc/vnode.h
View File

@ -210,12 +210,16 @@ struct SMetaEntry {
struct {
int64_t ctime;
int32_t ttlDays;
int32_t commentLen;
char *comment;
tb_uid_t suid;
uint8_t *pTags;
} ctbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
int32_t commentLen;
char *comment;
int32_t ncid; // next column id
SSchemaWrapper schemaRow;
} ntbEntry;

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -87,6 +87,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p
int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids);
int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
@ -105,6 +106,7 @@ int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
void* metaGetIdx(SMeta* pMeta);
void* metaGetIvtIdx(SMeta* pMeta);
int metaTtlSmaller(SMeta *pMeta, uint64_t time, SArray *uidList);
int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);

17
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -29,12 +29,20 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
if (pME->ctbEntry.commentLen > 0){
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1;
}
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
if (pME->ntbEntry.commentLen > 0){
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1;
}
if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1;
} else if (pME->type == TSDB_TSMA_TABLE) {
@ -61,12 +69,21 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
if (pME->ctbEntry.commentLen > 0){
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0)
return -1;
}
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO)
debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
if (pME->ntbEntry.commentLen > 0){
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1;
}
if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1;
} else if (pME->type == TSDB_TSMA_TABLE) {

36
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -95,7 +95,7 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
metaULock(pMeta);
return 0;
return uid;
}
int metaReadNext(SMetaReader *pReader) {
@ -218,6 +218,40 @@ _err:
return NULL;
}
int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList){
metaRLock(pMeta);
TBC * pCur;
int ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL);
if (ret < 0) {
metaULock(pMeta);
return ret;
}
STtlIdxKey ttlKey = {0};
ttlKey.dtime = ttl;
ttlKey.uid = INT64_MAX;
int c = 0;
tdbTbcMoveTo(pCur, &ttlKey, sizeof(ttlKey), &c);
if (c < 0) {
tdbTbcMoveToPrev(pCur);
}
void *pKey = NULL;
int kLen = 0;
while(1){
ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL);
if (ret < 0) {
break;
}
ttlKey = *(STtlIdxKey*)pKey;
taosArrayPush(uidList, &ttlKey.uid);
}
tdbTbcClose(pCur);
tdbFree(pKey);
return 0;
}
struct SMCtbCursor {
SMeta * pMeta;
TBC * pCur;

168
source/dnode/vnode/src/meta/metaTable.c
View File

@ -320,11 +320,15 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
if (me.type == TSDB_CHILD_TABLE) {
me.ctbEntry.ctime = pReq->ctime;
me.ctbEntry.ttlDays = pReq->ttl;
me.ctbEntry.commentLen = pReq->commentLen;
me.ctbEntry.comment = pReq->comment;
me.ctbEntry.suid = pReq->ctb.suid;
me.ctbEntry.pTags = pReq->ctb.pTag;
} else {
me.ntbEntry.ctime = pReq->ctime;
me.ntbEntry.ttlDays = pReq->ttl;
me.ntbEntry.commentLen = pReq->commentLen;
me.ntbEntry.comment = pReq->comment;
me.ntbEntry.schemaRow = pReq->ntb.schemaRow;
me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1;
}
@ -359,7 +363,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
metaDropTableByUid(pMeta, uid, &type);
metaULock(pMeta);
if (type == TSDB_CHILD_TABLE && tbUids) {
if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) {
taosArrayPush(tbUids, &uid);
}
@ -367,16 +371,57 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
return 0;
}
int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
metaWLock(pMeta);
int ret = metaTtlSmaller(pMeta, ttl, tbUids);
if(ret != 0){
return ret;
}
for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i);
metaDropTableByUid(pMeta, *uid, NULL);
}
metaULock(pMeta);
return 0;
}
static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
int32_t ttlDays;
int64_t ctime;
if (pME->type == TSDB_CHILD_TABLE) {
ctime = pME->ctbEntry.ctime;
ttlDays = pME->ctbEntry.ttlDays;
} else if (pME->type == TSDB_NORMAL_TABLE) {
ctime = pME->ntbEntry.ctime;
ttlDays = pME->ntbEntry.ttlDays;
} else {
ASSERT(0);
}
if (ttlDays <= 0) return;
ttlKey->dtime = ctime / 1000 + ttlDays * 24 * 60 * 60;
// ttlKey->dtime = ctime / 1000 + ttlDays;
ttlKey->uid = pME->uid;
}
static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
STtlIdxKey ttlKey = {0};
metaBuildTtlIdxKey(&ttlKey, pME);
if(ttlKey.dtime == 0) return 0;
return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), &pMeta->txn);
}
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
void * pData = NULL;
int nData = 0;
int rc = 0;
int64_t version;
SMetaEntry e = {0};
SDecoder dc = {0};
rc = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData);
version = *(int64_t *)pData;
int64_t version = *(int64_t *)pData;
tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData);
@ -388,15 +433,17 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn);
tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn);
tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
if (e.type == TSDB_CHILD_TABLE) {
tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn);
} else if (e.type == TSDB_NORMAL_TABLE) {
// drop schema.db (todo)
// drop ttl.idx (todo)
} else if (e.type == TSDB_SUPER_TABLE) {
// drop schema.db (todo)
}
metaError("ttl drop table:%s", e.name);
tDecoderClear(&dc);
tdbFree(pData);
@ -545,16 +592,20 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp);
if (entry.pBuf) taosMemoryFree(entry.pBuf);
if (pNewSchema) taosMemoryFree(pNewSchema);
tDecoderClear(&dc);
tdbTbcClose(pTbDbc);
tdbTbcClose(pUidIdxc);
tDecoderClear(&dc);
return 0;
_err:
tDecoderClear(&dc);
if (entry.pBuf) taosMemoryFree(entry.pBuf);
tdbTbcClose(pTbDbc);
tdbTbcClose(pUidIdxc);
tDecoderClear(&dc);
return -1;
}
@ -706,8 +757,87 @@ _err:
}
static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
// TODO
ASSERT(0);
void * pVal = NULL;
int nVal = 0;
const void * pData = NULL;
int nData = 0;
int ret = 0;
tb_uid_t uid;
int64_t oversion;
SMetaEntry entry = {0};
int c = 0;
// search name index
ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) {
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1;
}
uid = *(tb_uid_t *)pVal;
tdbFree(pVal);
pVal = NULL;
// search uid index
TBC *pUidIdxc = NULL;
tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
ASSERT(c == 0);
tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
oversion = *(int64_t *)pData;
// search table.db
TBC *pTbDbc = NULL;
tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(c == 0);
tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
// get table entry
SDecoder dc = {0};
entry.pBuf = taosMemoryMalloc(nData);
memcpy(entry.pBuf, pData, nData);
tDecoderInit(&dc, entry.pBuf, nData);
ret = metaDecodeEntry(&dc, &entry);
ASSERT(ret == 0);
entry.version = version;
metaWLock(pMeta);
// build SMetaEntry
if (entry.type == TSDB_CHILD_TABLE) {
if(pAlterTbReq->updateTTL) {
metaDeleteTtlIdx(pMeta, &entry);
entry.ctbEntry.ttlDays = pAlterTbReq->newTTL;
metaUpdateTtlIdx(pMeta, &entry);
}
if(pAlterTbReq->newCommentLen >= 0) {
entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen;
entry.ctbEntry.comment = pAlterTbReq->newComment;
}
} else {
if(pAlterTbReq->updateTTL) {
metaDeleteTtlIdx(pMeta, &entry);
entry.ntbEntry.ttlDays = pAlterTbReq->newTTL;
metaUpdateTtlIdx(pMeta, &entry);
}
if(pAlterTbReq->newCommentLen >= 0) {
entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen;
entry.ntbEntry.comment = pAlterTbReq->newComment;
}
}
// save to table db
metaSaveToTbDb(pMeta, &entry);
tdbTbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0);
metaULock(pMeta);
tdbTbcClose(pTbDbc);
tdbTbcClose(pUidIdxc);
tDecoderClear(&dc);
if (entry.pBuf) taosMemoryFree(entry.pBuf);
return 0;
}
@ -786,25 +916,9 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
}
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
int32_t ttlDays;
int64_t ctime;
STtlIdxKey ttlKey;
if (pME->type == TSDB_CHILD_TABLE) {
ctime = pME->ctbEntry.ctime;
ttlDays = pME->ctbEntry.ttlDays;
} else if (pME->type == TSDB_NORMAL_TABLE) {
ctime = pME->ntbEntry.ctime;
ttlDays = pME->ntbEntry.ttlDays;
} else {
ASSERT(0);
}
if (ttlDays <= 0) return 0;
ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
ttlKey.uid = pME->uid;
STtlIdxKey ttlKey = {0};
metaBuildTtlIdxKey(&ttlKey, pME);
if(ttlKey.dtime == 0) return 0;
return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
}

26
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -26,6 +26,7 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *
static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t code = 0;
@ -105,7 +106,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
int32_t len;
int32_t ret;
vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
vError("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
version);
pVnode->state.applied = version;
@ -134,6 +135,9 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
case TDMT_VND_DROP_TABLE:
if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_TTL_TABLE:
if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_SMA: {
if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
} break;
@ -300,6 +304,26 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
pMetaRsp->precision = pVnode->config.tsdbCfg.precision;
}
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp){
SArray *tbUids = taosArrayInit(8, sizeof(int64_t));
if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
int32_t t = ntohl(*(int32_t*)pReq);
vError("rec ttl time:%d", t);
int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
if(ret != 0){
goto end;
}
if(taosArrayGetSize(tbUids) > 0){
tqUpdateTbUidList(pVnode->pTq, tbUids, false);
}
end:
taosArrayDestroy(tbUids);
return ret;
}
static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SDecoder coder;

6
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -353,8 +353,8 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c
SyncIndex beginIndex = SYNC_INDEX_INVALID;
char logBuf[256] = {0};
if (pFsm->FpGetSnapshot != NULL) {
(*pFsm->FpGetSnapshot)(pFsm, &snapshot);
if (pFsm->FpGetSnapshotInfo != NULL) {
(*pFsm->FpGetSnapshotInfo)(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex;
}
@ -416,7 +416,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
pFsm->FpCommitCb = vnodeSyncCommitMsg;
pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot;
pFsm->FpRestoreFinishCb = NULL;
pFsm->FpReConfigCb = vnodeSyncReconfig;
pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead;

4
source/libs/command/src/explain.c
View File

@ -662,8 +662,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (pIndefNode->pVectorFuncs) {
EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIndefNode->pVectorFuncs->length);
if (pIndefNode->pFuncs) {
EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIndefNode->pFuncs->length);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIndefNode->node.pOutputDataBlockDesc->totalRowSize);

15
source/libs/executor/inc/executorimpl.h
View File

@ -425,6 +425,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
SArray* pChildren;
SSDataBlock* pUpdateRes;
SPhysiNode* pPhyNode; // create new child
bool isFinal;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo {
@ -549,14 +550,18 @@ typedef struct SStreamSessionAggOperatorInfo {
SGroupResInfo groupResInfo;
int64_t gap; // session window gap
int32_t primaryTsIndex; // primary timestamp slot id
int32_t endTsIndex; // window end timestamp slot id
int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
SSDataBlock* pWinBlock; // window result
SqlFunctionCtx* pDummyCtx; // for combine
SSDataBlock* pDelRes;
SSDataBlock* pDelRes; // delete result
SSDataBlock* pUpdateRes; // update window
SHashObj* pStDeleted;
void* pDelIterator;
SArray* pChildren; // cache for children's result; final stream operator
SPhysiNode* pPhyNode; // create new child
bool isFinal;
} SStreamSessionAggOperatorInfo;
typedef struct STimeSliceOperatorInfo {
@ -816,10 +821,10 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t size);
SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize);
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex);
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap,
SHashObj* pStDeleted);
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex);
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs,
TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted);
bool functionNeedToExecute(SqlFunctionCtx* pCtx);
int32_t compareTimeWindow(const void* p1, const void* p2, const void* param);

2
source/libs/executor/src/executorimpl.c
View File

@ -3716,7 +3716,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode;
int32_t numOfExpr = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pVectorFuncs, NULL, &numOfExpr);
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pFuncs, NULL, &numOfExpr);
if (pPhyNode->pExprs != NULL) {
SExprSupp* pSup1 = &pInfo->scalarSup;

51
source/libs/executor/src/scanoperator.c
View File

@ -740,11 +740,12 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
SStreamAggSupporter* pAggSup = pInfo->sessionSup.pStreamAggSup;
int64_t gap = pInfo->sessionSup.gap;
int32_t winIndex = 0;
SResultWindowInfo* pCurWin =
getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], pSDB->info.groupId, gap, &winIndex);
SResultWindowInfo* pCurWin =
getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN,
pSDB->info.groupId, gap, &winIndex);
win = pCurWin->win;
pInfo->updateResIndex +=
updateSessionWindowInfo(pCurWin, tsCols, pSDB->info.rows, pInfo->updateResIndex, gap, NULL);
updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL);
} else {
win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval,
pInfo->interval.precision, NULL);
@ -1402,12 +1403,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
pColInfoData = taosArrayGet(p->pDataBlock, 6);
colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false);
// table comment
// todo: set the correct comment
pColInfoData = taosArrayGet(p->pDataBlock, 8);
colDataAppendNULL(pColInfoData, numOfRows);
char str[256] = {0};
int32_t tableType = pInfo->pCur->mr.me.type;
if (tableType == TSDB_CHILD_TABLE) {
// create time
@ -1424,11 +1419,25 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false);
// super table name
STR_TO_VARSTR(str, mr.me.name);
STR_TO_VARSTR(n, mr.me.name);
pColInfoData = taosArrayGet(p->pDataBlock, 4);
colDataAppend(pColInfoData, numOfRows, str, false);
colDataAppend(pColInfoData, numOfRows, n, false);
metaReaderClear(&mr);
// table comment
pColInfoData = taosArrayGet(p->pDataBlock, 8);
if(pInfo->pCur->mr.me.ctbEntry.commentLen > 0) {
char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment);
colDataAppend(pColInfoData, numOfRows, comment, false);
}else if(pInfo->pCur->mr.me.ctbEntry.commentLen == 0) {
char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(comment, "");
colDataAppend(pColInfoData, numOfRows, comment, false);
}else{
colDataAppendNULL(pColInfoData, numOfRows);
}
// uid
pColInfoData = taosArrayGet(p->pDataBlock, 5);
colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false);
@ -1437,7 +1446,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
pColInfoData = taosArrayGet(p->pDataBlock, 7);
colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false);
STR_TO_VARSTR(str, "CHILD_TABLE");
STR_TO_VARSTR(n, "CHILD_TABLE");
} else if (tableType == TSDB_NORMAL_TABLE) {
// create time
pColInfoData = taosArrayGet(p->pDataBlock, 2);
@ -1451,6 +1460,20 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
pColInfoData = taosArrayGet(p->pDataBlock, 4);
colDataAppendNULL(pColInfoData, numOfRows);
// table comment
pColInfoData = taosArrayGet(p->pDataBlock, 8);
if(pInfo->pCur->mr.me.ntbEntry.commentLen > 0) {
char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment);
colDataAppend(pColInfoData, numOfRows, comment, false);
}else if(pInfo->pCur->mr.me.ntbEntry.commentLen == 0) {
char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(comment, "");
colDataAppend(pColInfoData, numOfRows, comment, false);
}else{
colDataAppendNULL(pColInfoData, numOfRows);
}
// uid
pColInfoData = taosArrayGet(p->pDataBlock, 5);
colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false);
@ -1459,11 +1482,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
pColInfoData = taosArrayGet(p->pDataBlock, 7);
colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false);
STR_TO_VARSTR(str, "NORMAL_TABLE");
STR_TO_VARSTR(n, "NORMAL_TABLE");
}
pColInfoData = taosArrayGet(p->pDataBlock, 9);
colDataAppend(pColInfoData, numOfRows, str, false);
colDataAppend(pColInfoData, numOfRows, n, false);
if (++numOfRows >= pOperator->resultInfo.capacity) {
break;

327
source/libs/executor/src/timewindowoperator.c
View File

@ -24,6 +24,8 @@ typedef enum SResultTsInterpType {
RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;
#define IS_FINAL_OP(op) ((op)->isFinal)
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);
@ -682,6 +684,13 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
}
}
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock));
taosArrayPush(blocks, pBlock);
blockDebugShowData(blocks, flag);
taosArrayDestroy(blocks);
}
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
@ -2054,8 +2063,6 @@ _error:
return NULL;
}
bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
SExecTaskInfo* pTaskInfo) {
for (int32_t k = 0; k < numOfOutput; ++k) {
@ -2129,7 +2136,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
pInfo->interval.precision, NULL);
while (1) {
if (isFinalInterval(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) {
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
taosArrayPush(pUpWins, &nextWin);
@ -2150,9 +2157,6 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
saveResultRow(pResult, tableGroupId, pUpdated);
}
// window start(end) key interpolation
// doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pSup->pCtx, pResult, &nextWin, startPos,
// forwardRows);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
@ -2214,8 +2218,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
if (isFinalInterval(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
if (!isFinalInterval(pInfo)) {
if (IS_FINAL_OP(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
if (!IS_FINAL_OP(pInfo)) {
// semi interval operator clear disk buffer
clearStreamIntervalOperator(pInfo);
}
@ -2239,7 +2243,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs,
pBlock, pUpWins);
if (isFinalInterval(pInfo)) {
if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SIntervalAggOperatorInfo* pChildInfo = pChildOp->info;
@ -2256,14 +2260,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
taosArrayDestroy(pUpWins);
break;
} else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo)) {
} else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
continue;
}
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated);
if (isFinalInterval(pInfo)) {
if (IS_FINAL_OP(pInfo)) {
int32_t chIndex = getChildIndex(pBlock);
int32_t size = taosArrayGetSize(pInfo->pChildren);
// if chIndex + 1 - size > 0, add new child
@ -2283,7 +2287,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
if (isFinalInterval(pInfo)) {
if (IS_FINAL_OP(pInfo)) {
closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
}
@ -2356,7 +2360,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
}
}
// semi interval operator does not catch result
if (!isFinalInterval(pInfo)) {
if (!IS_FINAL_OP(pInfo)) {
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
}
pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
@ -2364,8 +2368,15 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
pOperator->name = "StreamFinalIntervalOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
pInfo->isFinal = true;
pOperator->name = "StreamFinalIntervalOperator";
} else {
pInfo->isFinal = false;
pOperator->name = "StreamSemiIntervalOperator";
}
pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true;
pOperator->status = OP_NOT_OPENED;
pOperator->exprSupp.pExprInfo = pExprInfo;
@ -2455,7 +2466,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@ -2491,7 +2501,10 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
initResultRowInfo(&pInfo->binfo.resultRowInfo);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
pInfo->primaryTsIndex = tsSlotId;
pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
if (pSessionNode->window.pTsEnd) {
pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
}
pInfo->gap = pSessionNode->gap;
pInfo->binfo.pRes = pResBlock;
pInfo->order = TSDB_ORDER_ASC;
@ -2501,6 +2514,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pInfo->pDelRes = createOneDataBlock(pResBlock, false);
blockDataEnsureCapacity(pInfo->pDelRes, 64);
pInfo->pChildren = NULL;
pInfo->isFinal = false;
pInfo->pPhyNode = pPhyNode;
pOperator->name = "StreamSessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@ -2513,8 +2528,10 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
aggEncodeResultRow, aggDecodeResultRow, NULL);
pOperator->pTaskInfo = pTaskInfo;
initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
code = appendDownstream(pOperator, &downstream, 1);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
code = appendDownstream(pOperator, &downstream, 1);
}
return pOperator;
_error:
@ -2564,21 +2581,22 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
return pWinInfos;
}
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex) {
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex) {
SArray* pWinInfos = getWinInfos(pAggSup, groupId);
pAggSup->pCurWins = pWinInfos;
int32_t size = taosArrayGetSize(pWinInfos);
if (size == 0) {
*pIndex = 0;
return addNewSessionWindow(pWinInfos, ts);
return addNewSessionWindow(pWinInfos, startTs);
}
// find the first position which is smaller than the key
int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getSessionWindowEndkey);
int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
SResultWindowInfo* pWin = NULL;
if (index >= 0) {
pWin = taosArrayGet(pWinInfos, index);
if (isInWindow(pWin, ts, gap)) {
if (isInWindow(pWin, startTs, gap)) {
*pIndex = index;
return pWin;
}
@ -2586,34 +2604,40 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts,
if (index + 1 < size) {
pWin = taosArrayGet(pWinInfos, index + 1);
if (isInWindow(pWin, ts, gap)) {
if (isInWindow(pWin, startTs, gap)) {
*pIndex = index + 1;
return pWin;
} else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
*pIndex = index;
return pWin;
}
}
if (index == size - 1) {
*pIndex = taosArrayGetSize(pWinInfos);
return addNewSessionWindow(pWinInfos, ts);
return addNewSessionWindow(pWinInfos, startTs);
}
*pIndex = index + 1;
return insertNewSessionWindow(pWinInfos, ts, index + 1);
return insertNewSessionWindow(pWinInfos, startTs, index + 1);
}
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap,
SHashObj* pStDeleted) {
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs,
TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) {
for (int32_t i = start; i < rows; ++i) {
if (!isInWindow(pWinInfo, pTs[i], gap)) {
if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap)) ) {
return i - start;
}
if (pWinInfo->win.skey > pTs[i]) {
if (pWinInfo->win.skey > pStartTs[i]) {
if (pStDeleted && pWinInfo->isOutput) {
taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
pWinInfo->isOutput = false;
}
pWinInfo->win.skey = pTs[i];
pWinInfo->win.skey = pStartTs[i];
}
pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
if (pEndTs) {
pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
}
pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pTs[i]);
}
return rows - start;
}
@ -2666,7 +2690,7 @@ static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStre
if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, true);
updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
return TSDB_CODE_SUCCESS;
@ -2733,8 +2757,8 @@ typedef struct SWinRes {
uint64_t groupId;
} SWinRes;
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
SHashObj* pStDeleted) {
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock,
SHashObj* pStUpdated, SHashObj* pStDeleted, bool hasEndTs) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
bool masterScan = true;
@ -2745,13 +2769,21 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
int32_t step = 1;
bool ascScan = true;
TSKEY* tsCols = NULL;
TSKEY* startTsCols = NULL;
TSKEY* endTsCols = NULL;
SResultRow* pResult = NULL;
int32_t winRows = 0;
if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
startTsCols = (int64_t*) pStartTsCol->pData;
SColumnInfoData* pEndTsCol = NULL;
if (hasEndTs) {
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
} else {
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
}
endTsCols = (int64_t*) pEndTsCol->pData;
} else {
return;
}
@ -2759,22 +2791,21 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
for (int32_t i = 0; i < pSDataBlock->info.rows;) {
int32_t winIndex = 0;
SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], groupId, gap, &winIndex);
winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i],
endTsCols[i], groupId, gap, &winIndex);
winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols,
pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
// window start(end) key interpolation
// doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pSup->pCtx, pResult, &nextWin, startPos,
// forwardRows,
// pInfo->order, false);
int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
if (winNum > 0) {
compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
}
pCurWin->isClosed = false;
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId};
code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
if (code != TSDB_CODE_SUCCESS) {
@ -2793,8 +2824,8 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup,
int32_t step = 0;
for (int32_t i = 0; i < pBlock->info.rows; i += step) {
int32_t winIndex = 0;
SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pBlock->info.groupId, gap, &winIndex);
step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL);
SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex);
step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL);
ASSERT(isInWindow(pCurWin, tsCols[i], gap));
doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
if (result) {
@ -2861,7 +2892,10 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
SArray* pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
int32_t chWinSize = taosArrayGetSize(pChWins);
int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
for (int32_t k = index; k > 0 && k < chWinSize; k++) {
if (index < 0) {
index = 0;
}
for (int32_t k = index; k < chWinSize; k++) {
SResultWindowInfo* pcw = taosArrayGet(pChWins, k);
if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) {
SResultRow* pChResult = NULL;
@ -2876,11 +2910,9 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
}
}
bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }
typedef SResultWindowInfo* (*__get_win_info_)(void*);
SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed,
__get_win_info_ fn) {
@ -2928,14 +2960,13 @@ int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_
}
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SExprSupp* pSup = &pOperator->exprSupp;
SExprSupp* pSup = &pOperator->exprSupp;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
if (pOperator->status == OP_RES_TO_RETURN) {
TSKEY maxTs = INT64_MIN;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
} else if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
return pInfo->pDelRes;
@ -2960,8 +2991,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
if (pBlock->info.type == STREAM_REPROCESS) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, pInfo->gap, pWins);
if (isFinalSession(pInfo)) {
int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock
if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, 0, pChildOp->exprSupp.numOfExprs,
@ -2971,25 +3002,139 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
taosArrayDestroy(pWins);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getSessionWinInfo);
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
continue;
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
if (isFinalSession(pInfo)) {
int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock
SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
doStreamSessionAggImpl(pOperator, pBlock, NULL, NULL);
doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
if (IS_FINAL_OP(pInfo)) {
int32_t chIndex = getChildIndex(pBlock);
int32_t size = taosArrayGetSize(pInfo->pChildren);
// if chIndex + 1 - size > 0, add new child
for (int32_t i = 0; i < chIndex + 1 - size; i++) {
SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
if (!pChildOp) {
longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
taosArrayPush(pInfo->pChildren, &pChildOp);
}
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
}
doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
maxTs = TMAX(maxTs, pBlock->info.window.ekey);
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
// restore the value
pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
getSessionWinInfo);
getResWinForSession);
copyUpdateResult(pStUpdated, pUpdated);
taosHashCleanup(pStUpdated);
finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
void **pIte = NULL;
while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
SArray *pWins = (SArray *) (*pIte);
int32_t size = taosArrayGetSize(pWins);
for (int32_t i = 0; i < size; i++) {
SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
pWin->pos.pageId = -1;
pWin->pos.offset = -1;
}
}
clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
}
static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
int32_t size = taosArrayGetSize(pWins);
for (int32_t i = 0; i < size; i++) {
SResultWindowInfo* pWin = taosArrayGet(pWins, i);
taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
}
}
static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
TSKEY maxTs = INT64_MIN;
SExprSupp* pSup = &pOperator->exprSupp;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
} else if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
if (pInfo->pUpdateRes->info.rows == 0) {
// semi interval operator clear disk buffer
clearStreamSessionOperator(pInfo);
return NULL;
}
// process the rest of the data
pOperator->status = OP_OPENED;
return pInfo->pUpdateRes;
}
return pInfo->binfo.pRes;
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
SOperatorInfo* downstream = pOperator->pDownstream[0];
SArray* pUpdated = taosArrayInit(16, POINTER_BYTES);
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
clearUpdateDataBlock(pInfo->pUpdateRes);
break;
}
if (pBlock->info.type == STREAM_REPROCESS) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins);
removeSessionResults(pStUpdated, pWins);
taosArrayDestroy(pWins);
copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
break;
} else if (pBlock->info.type == STREAM_GET_ALL) {
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
continue;
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false);
maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
// restore the value
pOperator->status = OP_RES_TO_RETURN;
// semi operator
// closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
// getResWinForSession);
copyUpdateResult(pStUpdated, pUpdated);
taosHashCleanup(pStUpdated);
@ -3002,6 +3147,15 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
if (pInfo->pUpdateRes->info.rows == 0) {
return NULL;
}
// process the rest of the data
pOperator->status = OP_OPENED;
return pInfo->pUpdateRes;
}
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@ -3012,17 +3166,32 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
if (pOperator == NULL) {
goto _error;
}
pOperator->name = "StreamFinalSessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
pInfo->pChildren = taosArrayInit(8, sizeof(void*));
for (int32_t i = 0; i < numOfChild; i++) {
SOperatorInfo* pChild =
createStreamSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo);
if (pChild == NULL) {
goto _error;
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
pInfo->isFinal = true;
pOperator->name = "StreamSessionFinalAggOperator";
} else {
pInfo->isFinal = false;
pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
pOperator->name = "StreamSessionSemiAggOperator";
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
aggEncodeResultRow, aggDecodeResultRow, NULL);
}
pOperator->operatorType = pPhyNode->type;
if (numOfChild > 0) {
pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
for (int32_t i = 0; i < numOfChild; i++) {
SOperatorInfo* pChild =
createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
if (pChild == NULL) {
goto _error;
}
taosArrayPush(pInfo->pChildren, &pChild);
}
taosArrayPush(pInfo->pChildren, &pChild);
}
return pOperator;
@ -3331,7 +3500,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
pSeUpdated, pInfo->pSeDeleted);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getStateWinInfo);
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
continue;
}
@ -3344,7 +3513,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
getStateWinInfo);
getResWinForState);
copyUpdateResult(pSeUpdated, pUpdated);
taosHashCleanup(pSeUpdated);

30
source/libs/function/inc/builtins.h
View File

@ -24,22 +24,24 @@ extern "C" {
typedef int32_t (*FTranslateFunc)(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
typedef int32_t (*FCreateMergeFuncParameters)(SNodeList* pRawParameters, SNode* pPartialRes, SNodeList** pParameters);
typedef struct SBuiltinFuncDefinition {
const char* name;
EFunctionType type;
uint64_t classification;
FTranslateFunc translateFunc;
FFuncDataRequired dataRequiredFunc;
FExecGetEnv getEnvFunc;
FExecInit initFunc;
FExecProcess processFunc;
FScalarExecProcess sprocessFunc;
FExecFinalize finalizeFunc;
FExecProcess invertFunc;
FExecCombine combineFunc;
const char* pPartialFunc;
const char* pMergeFunc;
const char* name;
EFunctionType type;
uint64_t classification;
FTranslateFunc translateFunc;
FFuncDataRequired dataRequiredFunc;
FExecGetEnv getEnvFunc;
FExecInit initFunc;
FExecProcess processFunc;
FScalarExecProcess sprocessFunc;
FExecFinalize finalizeFunc;
FExecProcess invertFunc;
FExecCombine combineFunc;
const char* pPartialFunc;
const char* pMergeFunc;
FCreateMergeFuncParameters createMergeParaFuc;
} SBuiltinFuncDefinition;
extern const SBuiltinFuncDefinition funcMgtBuiltins[];

2
source/libs/function/inc/functionMgtInt.h
View File

@ -51,8 +51,6 @@ extern "C" {
#define FUNC_UDF_ID_START 5000
extern const int funcMgtUdfNum;
#ifdef __cplusplus
}
#endif

13
source/libs/function/src/builtins.c
View File

@ -419,6 +419,14 @@ static int32_t translateTopBot(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return TSDB_CODE_SUCCESS;
}
int32_t topCreateMergePara(SNodeList* pRawParameters, SNode* pPartialRes, SNodeList** pParameters) {
int32_t code = nodesListMakeAppend(pParameters, pPartialRes);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(*pParameters, nodesCloneNode(nodesListGetNode(pRawParameters, 1)));
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateTopBotImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
@ -1728,7 +1736,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = topBotFinalize,
.combineFunc = topCombine,
.pPartialFunc = "_top_partial",
.pMergeFunc = "_top_merge"
.pMergeFunc = "_top_merge",
// .createMergeParaFuc = topCreateMergePara
},
{
.name = "_top_partial",
@ -1872,7 +1881,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "interp",
.type = FUNCTION_TYPE_INTERP,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC,
.classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC,
.translateFunc = translateFirstLast,
.getEnvFunc = getSelectivityFuncEnv,
.initFunc = functionSetup,

122
source/libs/function/src/builtinsimpl.c
View File

@ -794,8 +794,8 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data);
avgTransferInfo(pInputInfo, pInfo);
@ -908,9 +908,9 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getAvgInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getAvgInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes);
@ -1418,7 +1418,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) {
for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
colDataAppendNULL(pDstCol, rowIndex);
@ -1481,11 +1481,13 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
if (pSBuf->assign && ((((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign)) {
*(double*)&pDBuf->v = *(double*)&pSBuf->v;
replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos);
pDBuf->assign = true;
}
} else {
if (pSBuf->assign && (((pDBuf->v < pSBuf->v) ^ isMinFunc) || !pDBuf->assign)) {
pDBuf->v = pSBuf->v;
replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos);
pDBuf->assign = true;
}
}
pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
@ -1663,8 +1665,8 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {
SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data);
stddevTransferInfo(pInputInfo, pInfo);
@ -1745,10 +1747,10 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
if (IS_INTEGER_TYPE(type)) {
avg = pStddevRes->isum / ((double)pStddevRes->count);
pStddevRes->result = sqrt(pStddevRes->quadraticISum / ((double)pStddevRes->count) - avg * avg);
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticISum / ((double)pStddevRes->count) - avg * avg));
} else {
avg = pStddevRes->dsum / ((double)pStddevRes->count);
pStddevRes->result = sqrt(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg);
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg));
}
return functionFinalize(pCtx, pBlock);
@ -1756,9 +1758,9 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getStddevInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getStddevInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes);
@ -2360,12 +2362,10 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
return TSDB_CODE_SUCCESS;
}
int32_t getFirstLastInfoSize(int32_t resBytes) {
return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t);
}
int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); }
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pNode = (SColumnNode *)nodesListGetNode(pFunc->pParameterList, 0);
SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t);
return true;
}
@ -2390,14 +2390,14 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = 0;
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
int32_t type = pInputCol->info.type;
int32_t type = pInputCol->info.type;
int32_t bytes = pInputCol->info.bytes;
pInfo->bytes = bytes;
pInfo->bytes = bytes;
// All null data column, return directly.
if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
@ -2488,14 +2488,14 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = 0;
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
int32_t type = pInputCol->info.type;
int32_t type = pInputCol->info.type;
int32_t bytes = pInputCol->info.bytes;
pInfo->bytes = bytes;
pInfo->bytes = bytes;
// All null data column, return directly.
if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
@ -2567,8 +2567,8 @@ static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput,
return;
}
pOutput->bytes = pInput->bytes;
TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes);
TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes);
TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes);
TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes);
if (pOutput->hasResult) {
if (isFirst) {
if (*tsIn > *tsOut) {
@ -2586,16 +2586,16 @@ static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput,
return;
}
static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx *pCtx, bool isFirstQuery) {
static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
SColumnInfoData* pCol = pInput->pData[0];
ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
SFirstLastRes* pInputInfo = (SFirstLastRes *)varDataVal(data);
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data);
firstLastTransferInfo(pInputInfo, pInfo, isFirstQuery);
@ -2604,13 +2604,9 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx *pCtx, bool isFirstQuer
return TSDB_CODE_SUCCESS;
}
int32_t firstFunctionMerge(SqlFunctionCtx *pCtx) {
return firstLastFunctionMergeImpl(pCtx, true);
}
int32_t firstFunctionMerge(SqlFunctionCtx* pCtx) { return firstLastFunctionMergeImpl(pCtx, true); }
int32_t lastFunctionMerge(SqlFunctionCtx *pCtx) {
return firstLastFunctionMergeImpl(pCtx, false);
}
int32_t lastFunctionMerge(SqlFunctionCtx* pCtx) { return firstLastFunctionMergeImpl(pCtx, false); }
int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
@ -2627,9 +2623,9 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getFirstLastInfoSize(pRes->bytes);
char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getFirstLastInfoSize(pRes->bytes);
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
memcpy(varDataVal(res), pRes, resultBytes);
varDataSetLen(res, resultBytes);
@ -3369,8 +3365,8 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) {
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data);
spreadTransferInfo(pInputInfo, pInfo);
@ -3390,9 +3386,9 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getSpreadInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getSpreadInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes);
@ -3484,8 +3480,8 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) {
SColumnInfoData* pCol = pInput->pData[0];
int32_t start = pInput->startRowIndex;
TSKEY* ptsList = (int64_t*)colDataGetData(pCol, 0);
int32_t start = pInput->startRowIndex;
TSKEY* ptsList = (int64_t*)colDataGetData(pCol, 0);
if (pCtx->order == TSDB_ORDER_DESC) {
if (pCtx->start.key == INT64_MIN) {
pInfo->max =
@ -5206,11 +5202,11 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
}
typedef struct SDerivInfo {
double prevValue; // previous value
TSKEY prevTs; // previous timestamp
bool ignoreNegative;// ignore the negative value
int64_t tsWindow; // time window for derivative
bool valueSet; // the value has been set already
double prevValue; // previous value
TSKEY prevTs; // previous timestamp
bool ignoreNegative; // ignore the negative value
int64_t tsWindow; // time window for derivative
bool valueSet; // the value has been set already
} SDerivInfo;
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
@ -5218,7 +5214,7 @@ bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
return true;
}
bool derivativeFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) {
bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
if (!functionSetup(pCtx, pResInfo)) {
return false; // not initialized since it has been initialized
}
@ -5226,25 +5222,25 @@ bool derivativeFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) {
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
pDerivInfo->ignoreNegative = pCtx->param[2].param.i;
pDerivInfo->prevTs = -1;
pDerivInfo->prevTs = -1;
pDerivInfo->tsWindow = pCtx->param[1].param.i;
pDerivInfo->valueSet = false;
return true;
}
int32_t derivativeFunction(SqlFunctionCtx *pCtx) {
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t derivativeFunction(SqlFunctionCtx* pCtx) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
SColumnInfoData* pInputCol = pInput->pData[0];
int32_t numOfElems = 0;
int32_t numOfElems = 0;
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
SColumnInfoData* pTsOutput = pCtx->pTsOutput;
int32_t i = pInput->startRowIndex;
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
double v = 0;
@ -5254,7 +5250,7 @@ int32_t derivativeFunction(SqlFunctionCtx *pCtx) {
continue;
}
char* d = (char*)pInputCol->pData + pInputCol->info.bytes * i;
char* d = (char*)pInputCol->pData + pInputCol->info.bytes * i;
GET_TYPED_DATA(v, double, pInputCol->info.type, d);
int32_t pos = pCtx->offset + numOfElems;
@ -5281,7 +5277,7 @@ int32_t derivativeFunction(SqlFunctionCtx *pCtx) {
continue;
}
char* d = (char*)pInputCol->pData + pInputCol->info.bytes * i;
char* d = (char*)pInputCol->pData + pInputCol->info.bytes * i;
GET_TYPED_DATA(v, double, pInputCol->info.type, d);
int32_t pos = pCtx->offset + numOfElems;
@ -5307,7 +5303,7 @@ int32_t derivativeFunction(SqlFunctionCtx *pCtx) {
return numOfElems;
}
int32_t interpFunction(SqlFunctionCtx *pCtx) {
int32_t interpFunction(SqlFunctionCtx* pCtx) {
#if 0
int32_t fillType = (int32_t) pCtx->param[2].i64;
//bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
@ -5398,4 +5394,4 @@ int32_t interpFunction(SqlFunctionCtx *pCtx) {
#endif
return TSDB_CODE_SUCCESS;
}
}

91
source/libs/function/src/functionMgt.c
View File

@ -70,11 +70,21 @@ int32_t fmFuncMgtInit() {
}
int32_t fmGetFuncInfo(SFunctionNode* pFunc, char* pMsg, int32_t msgLen) {
void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc->functionName, strlen(pFunc->functionName));
if (NULL != pVal) {
pFunc->funcId = *(int32_t*)pVal;
pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type;
return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pMsg, msgLen);
if (NULL != gFunMgtService.pFuncNameHashTable) {
void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc->functionName, strlen(pFunc->functionName));
if (NULL != pVal) {
pFunc->funcId = *(int32_t*)pVal;
pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type;
return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pMsg, msgLen);
}
return TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION;
}
for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) {
if (0 == strcmp(funcMgtBuiltins[i].name, pFunc->functionName)) {
pFunc->funcId = i;
pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type;
return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pMsg, msgLen);
}
}
return TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION;
}
@ -169,6 +179,13 @@ bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId
bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); }
bool fmIsInterpFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_INTERP == funcMgtBuiltins[funcId].type;
}
void fmFuncMgtDestroy() {
void* m = gFunMgtService.pFuncNameHashTable;
if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) {
@ -233,17 +250,7 @@ bool fmIsSameInOutType(int32_t funcId) {
static int32_t getFuncInfo(SFunctionNode* pFunc) {
char msg[64] = {0};
if (NULL != gFunMgtService.pFuncNameHashTable) {
return fmGetFuncInfo(pFunc, msg, sizeof(msg));
}
for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) {
if (0 == strcmp(funcMgtBuiltins[i].name, pFunc->functionName)) {
pFunc->funcId = i;
pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type;
return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, msg, sizeof(msg));
}
}
return TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION;
return fmGetFuncInfo(pFunc, msg, sizeof(msg));
}
static SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) {
@ -261,14 +268,14 @@ static SFunctionNode* createFunction(const char* pName, SNodeList* pParameterLis
return pFunc;
}
static SColumnNode* createColumnByFunc(const SFunctionNode* pFunc) {
static SNode* createColumnByFunc(const SFunctionNode* pFunc) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
}
strcpy(pCol->colName, pFunc->node.aliasName);
pCol->node.resType = pFunc->node.resType;
return pCol;
return (SNode*)pCol;
}
bool fmIsDistExecFunc(int32_t funcId) {
@ -296,21 +303,47 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
return TSDB_CODE_SUCCESS;
}
static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc,
SNodeList** pParameterList) {
SNode* pRes = createColumnByFunc(pPartialFunc);
if (NULL != funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc) {
return funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc(pSrcFunc->pParameterList, pRes, pParameterList);
} else {
return nodesListMakeStrictAppend(pParameterList, pRes);
}
}
static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc,
SFunctionNode** pMergeFunc) {
SNodeList* pParameterList = NULL;
nodesListMakeStrictAppend(&pParameterList, (SNode*)createColumnByFunc(pPartialFunc));
*pMergeFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList);
if (NULL == *pMergeFunc) {
SNodeList* pParameterList = NULL;
SFunctionNode* pFunc = NULL;
int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList);
if (TSDB_CODE_SUCCESS == code) {
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList);
if (NULL == pFunc) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
// overwrite function restype set by translate function
if (fmIsSameInOutType(pSrcFunc->funcId)) {
pFunc->node.resType = pSrcFunc->node.resType;
}
strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName);
}
if (TSDB_CODE_SUCCESS == code) {
*pMergeFunc = pFunc;
} else {
if (NULL != pFunc) {
pFunc->pParameterList = NULL;
nodesDestroyNode((SNode*)pFunc);
}
nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY;
}
// overwrite function restype set by translate function
if (fmIsSameInOutType(pSrcFunc->funcId)) {
(*pMergeFunc)->node.resType = pSrcFunc->node.resType;
}
strcpy((*pMergeFunc)->node.aliasName, pSrcFunc->node.aliasName);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc) {

17
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -423,6 +423,7 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD
COPY_SCALAR_FIELD(slidingUnit);
COPY_SCALAR_FIELD(sessionGap);
CLONE_NODE_FIELD(pTspk);
CLONE_NODE_FIELD(pTsEnd);
CLONE_NODE_FIELD(pStateExpr);
COPY_SCALAR_FIELD(triggerType);
COPY_SCALAR_FIELD(watermark);
@ -454,7 +455,18 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi
static SNode* logicIndefRowsFuncCopy(const SIndefRowsFuncLogicNode* pSrc, SIndefRowsFuncLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pVectorFuncs);
CLONE_NODE_LIST_FIELD(pFuncs);
return (SNode*)pDst;
}
static SNode* logicInterpFuncCopy(const SInterpFuncLogicNode* pSrc, SInterpFuncLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pFuncs);
COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
COPY_SCALAR_FIELD(interval);
COPY_SCALAR_FIELD(fillMode);
CLONE_NODE_FIELD(pFillValues);
CLONE_NODE_FIELD(pTimeSeries);
return (SNode*)pDst;
}
@ -522,6 +534,7 @@ static SNode* physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pD
CLONE_NODE_LIST_FIELD(pExprs);
CLONE_NODE_LIST_FIELD(pFuncs);
CLONE_NODE_FIELD(pTspk);
CLONE_NODE_FIELD(pTsEnd);
COPY_SCALAR_FIELD(triggerType);
COPY_SCALAR_FIELD(watermark);
COPY_SCALAR_FIELD(filesFactor);
@ -668,6 +681,8 @@ SNode* nodesCloneNode(const SNode* pNode) {
return logicPartitionCopy((const SPartitionLogicNode*)pNode, (SPartitionLogicNode*)pDst);
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
return logicIndefRowsFuncCopy((const SIndefRowsFuncLogicNode*)pNode, (SIndefRowsFuncLogicNode*)pDst);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return logicInterpFuncCopy((const SInterpFuncLogicNode*)pNode, (SInterpFuncLogicNode*)pDst);
case QUERY_NODE_LOGIC_SUBPLAN:
return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:

149
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -202,6 +202,8 @@ const char* nodesNodeName(ENodeType type) {
return "LogicPartition";
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
return "LogicIndefRowsFunc";
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return "LogicInterpFunc";
case QUERY_NODE_LOGIC_SUBPLAN:
return "LogicSubplan";
case QUERY_NODE_LOGIC_PLAN:
@ -258,6 +260,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiPartition";
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return "PhysiIndefRowsFunc";
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return "PhysiInterpFunc";
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return "PhysiDispatch";
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -935,14 +939,14 @@ static int32_t jsonToLogicPartitionNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkIndefRowsFuncLogicPlanVectorFuncs = "VectorFuncs";
static const char* jkIndefRowsFuncLogicPlanFuncs = "Funcs";
static int32_t logicIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) {
const SIndefRowsFuncLogicNode* pNode = (const SIndefRowsFuncLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkIndefRowsFuncLogicPlanVectorFuncs, pNode->pVectorFuncs);
code = nodeListToJson(pJson, jkIndefRowsFuncLogicPlanFuncs, pNode->pFuncs);
}
return code;
@ -953,7 +957,52 @@ static int32_t jsonToLogicIndefRowsFuncNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkIndefRowsFuncLogicPlanVectorFuncs, &pNode->pVectorFuncs);
code = jsonToNodeList(pJson, jkIndefRowsFuncLogicPlanFuncs, &pNode->pFuncs);
}
return code;
}
static const char* jkInterpFuncLogicPlanFuncs = "Funcs";
static const char* jkInterpFuncLogicPlanStartTime = "StartTime";
static const char* jkInterpFuncLogicPlanEndTime = "EndTime";
static const char* jkInterpFuncLogicPlanInterval = "Interval";
static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
const SInterpFuncLogicNode* pNode = (const SInterpFuncLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkInterpFuncLogicPlanFuncs, pNode->pFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanStartTime, pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanEndTime, pNode->timeRange.ekey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanInterval, pNode->interval);
}
return code;
}
static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) {
SInterpFuncLogicNode* pNode = (SInterpFuncLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkInterpFuncLogicPlanFuncs, &pNode->pFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanStartTime, &pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanEndTime, &pNode->timeRange.ekey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanInterval, &pNode->interval);
}
return code;
@ -1784,6 +1833,7 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
static const char* jkWindowPhysiPlanExprs = "Exprs";
static const char* jkWindowPhysiPlanFuncs = "Funcs";
static const char* jkWindowPhysiPlanTsPk = "TsPk";
static const char* jkWindowPhysiPlanTsEnd = "TsEnd";
static const char* jkWindowPhysiPlanTriggerType = "TriggerType";
static const char* jkWindowPhysiPlanWatermark = "Watermark";
static const char* jkWindowPhysiPlanFilesFactor = "FilesFactor";
@ -1801,6 +1851,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkWindowPhysiPlanTsPk, nodeToJson, pNode->pTspk);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkWindowPhysiPlanTsEnd, nodeToJson, pNode->pTsEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType);
}
@ -1827,6 +1880,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsPk, (SNode**)&pNode->pTspk);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsEnd, (SNode**)&pNode->pTsEnd);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code);
;
@ -2042,7 +2098,7 @@ static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) {
}
static const char* jkIndefRowsFuncPhysiPlanExprs = "Exprs";
static const char* jkIndefRowsFuncPhysiPlanVectorFuncs = "VectorFuncs";
static const char* jkIndefRowsFuncPhysiPlanFuncs = "Funcs";
static int32_t physiIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) {
const SIndefRowsFuncPhysiNode* pNode = (const SIndefRowsFuncPhysiNode*)pObj;
@ -2052,7 +2108,7 @@ static int32_t physiIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) {
code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanExprs, pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanVectorFuncs, pNode->pVectorFuncs);
code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanFuncs, pNode->pFuncs);
}
return code;
@ -2066,7 +2122,80 @@ static int32_t jsonToPhysiIndefRowsFuncNode(const SJson* pJson, void* pObj) {
code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanExprs, &pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanVectorFuncs, &pNode->pVectorFuncs);
code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanFuncs, &pNode->pFuncs);
}
return code;
}
static const char* jkInterpFuncPhysiPlanExprs = "Exprs";
static const char* jkInterpFuncPhysiPlanFuncs = "Funcs";
static const char* jkInterpFuncPhysiPlanStartTime = "StartTime";
static const char* jkInterpFuncPhysiPlanEndTime = "EndTime";
static const char* jkInterpFuncPhysiPlanInterval = "Interval";
static const char* jkInterpFuncPhysiPlanFillMode = "FillMode";
static const char* jkInterpFuncPhysiPlanFillValues = "FillValues";
static const char* jkInterpFuncPhysiPlanTimeSeries = "TimeSeries";
static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
const SInterpFuncPhysiNode* pNode = (const SInterpFuncPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkInterpFuncPhysiPlanExprs, pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkInterpFuncPhysiPlanFuncs, pNode->pFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanStartTime, pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanEndTime, pNode->timeRange.ekey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanInterval, pNode->interval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanFillMode, pNode->fillMode);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanFillValues, nodeToJson, pNode->pFillValues);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanTimeSeries, nodeToJson, pNode->pTimeSeries);
}
return code;
}
static int32_t jsonToPhysiInterpFuncNode(const SJson* pJson, void* pObj) {
SInterpFuncPhysiNode* pNode = (SInterpFuncPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkInterpFuncPhysiPlanExprs, &pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkInterpFuncPhysiPlanFuncs, &pNode->pFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanStartTime, &pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanEndTime, &pNode->timeRange.ekey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanInterval, &pNode->interval);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkInterpFuncPhysiPlanFillMode, pNode->fillMode, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkInterpFuncPhysiPlanFillValues, &pNode->pFillValues);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkInterpFuncPhysiPlanTimeSeries, &pNode->pTimeSeries);
}
return code;
@ -3968,6 +4097,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return logicPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
return logicIndefRowsFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return logicInterpFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_SUBPLAN:
return logicSubplanToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN:
@ -4013,6 +4144,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return physiIndefRowsFuncNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return physiInterpFuncNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return physiDispatchNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -4104,6 +4237,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicPartitionNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
return jsonToLogicIndefRowsFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return jsonToLogicInterpFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_SUBPLAN:
return jsonToLogicSubplan(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN:
@ -4149,6 +4284,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToPhysiPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return jsonToPhysiIndefRowsFuncNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return jsonToPhysiInterpFuncNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return jsonToPhysiDispatchNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DELETE:

24
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -250,6 +250,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SPartitionLogicNode));
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
return makeNode(type, sizeof(SIndefRowsFuncLogicNode));
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return makeNode(type, sizeof(SInterpFuncLogicNode));
case QUERY_NODE_LOGIC_SUBPLAN:
return makeNode(type, sizeof(SLogicSubplan));
case QUERY_NODE_LOGIC_PLAN:
@ -308,6 +310,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SPartitionPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return makeNode(type, sizeof(SIndefRowsFuncPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
return makeNode(type, sizeof(SInterpFuncLogicNode));
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return makeNode(type, sizeof(SDataDispatcherNode));
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -352,6 +356,7 @@ static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) {
nodesDestroyList(pNode->pExprs);
nodesDestroyList(pNode->pFuncs);
nodesDestroyNode(pNode->pTspk);
nodesDestroyNode(pNode->pTsEnd);
}
static void destroyScanPhysiNode(SScanPhysiNode* pNode) {
@ -525,6 +530,7 @@ void nodesDestroyNode(SNode* pNode) {
SCreateSubTableClause* pStmt = (SCreateSubTableClause*)pNode;
nodesDestroyList(pStmt->pSpecificTags);
nodesDestroyList(pStmt->pValsOfTags);
nodesDestroyNode((SNode*)pStmt->pOptions);
break;
}
case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
@ -718,6 +724,7 @@ void nodesDestroyNode(SNode* pNode) {
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyList(pLogicNode->pFuncs);
nodesDestroyNode(pLogicNode->pTspk);
nodesDestroyNode(pLogicNode->pTsEnd);
break;
}
case QUERY_NODE_LOGIC_PLAN_FILL: {
@ -742,7 +749,13 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: {
SIndefRowsFuncLogicNode* pLogicNode = (SIndefRowsFuncLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyList(pLogicNode->pVectorFuncs);
nodesDestroyList(pLogicNode->pFuncs);
break;
}
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: {
SInterpFuncLogicNode* pLogicNode = (SInterpFuncLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyList(pLogicNode->pFuncs);
break;
}
case QUERY_NODE_LOGIC_SUBPLAN: {
@ -845,7 +858,14 @@ void nodesDestroyNode(SNode* pNode) {
SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pExprs);
nodesDestroyList(pPhyNode->pVectorFuncs);
nodesDestroyList(pPhyNode->pFuncs);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: {
SInterpFuncPhysiNode* pPhyNode = (SInterpFuncPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pExprs);
nodesDestroyList(pPhyNode->pFuncs);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:

4
source/libs/parser/inc/parAst.h
View File

@ -109,6 +109,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode
SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
@ -118,6 +119,9 @@ SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving);
SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList);
SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit);
SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);

19
source/libs/parser/inc/sql.y
View File

@ -791,7 +791,7 @@ join_type(A) ::= INNER.
/************************************************ query_specification *************************************************/
query_specification(A) ::=
SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E)
partition_by_clause_opt(F) twindow_clause_opt(G)
partition_by_clause_opt(F) range_opt(J) every_opt(K) fill_opt(L) twindow_clause_opt(G)
group_by_clause_opt(H) having_clause_opt(I). {
A = createSelectStmt(pCxt, B, C, D);
A = addWhereClause(pCxt, A, E);
@ -799,6 +799,9 @@ query_specification(A) ::=
A = addWindowClauseClause(pCxt, A, G);
A = addGroupByClause(pCxt, A, H);
A = addHavingClause(pCxt, A, I);
A = addRangeClause(pCxt, A, J);
A = addEveryClause(pCxt, A, K);
A = addFillClause(pCxt, A, L);
}
%type set_quantifier_opt { bool }
@ -860,14 +863,20 @@ fill_mode(A) ::= NEXT.
group_by_clause_opt(A) ::= . { A = NULL; }
group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { A = B; }
%type group_by_list { SNodeList* }
%destructor group_by_list { nodesDestroyList($$); }
group_by_list(A) ::= expression(B). { A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); }
group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); }
%type group_by_list { SNodeList* }
%destructor group_by_list { nodesDestroyList($$); }
group_by_list(A) ::= expression(B). { A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); }
group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); }
having_clause_opt(A) ::= . { A = NULL; }
having_clause_opt(A) ::= HAVING search_condition(B). { A = B; }
range_opt(A) ::= . { A = NULL; }
range_opt(A) ::= RANGE NK_LP expression(B) NK_COMMA expression(C) NK_RP. { A = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); }
every_opt(A) ::= . { A = NULL; }
every_opt(A) ::= EVERY NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }
/************************************************ query_expression ****************************************************/
query_expression(A) ::=
query_expression_body(B)

49
source/libs/parser/src/parAstCreater.c
View File

@ -599,6 +599,11 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) {
return (SNode*)groupingSet;
}
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd) {
CHECK_PARSER_STATUS(pCxt);
return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt), pStart, pEnd);
}
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) {
CHECK_PARSER_STATUS(pCxt);
int32_t len = TMIN(sizeof(((SExprNode*)pNode)->aliasName) - 1, pAlias->n);
@ -671,6 +676,35 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) {
CHECK_PARSER_STATUS(pCxt);
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
((SSelectStmt*)pStmt)->pLimit = (SLimitNode*)pLimit;
} else {
((SSetOperator*)pStmt)->pLimit = pLimit;
}
return pStmt;
}
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange) {
CHECK_PARSER_STATUS(pCxt);
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
((SSelectStmt*)pStmt)->pRange = pRange;
}
return pStmt;
}
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery) {
CHECK_PARSER_STATUS(pCxt);
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
((SSelectStmt*)pStmt)->pEvery = pEvery;
}
return pStmt;
}
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) {
CHECK_PARSER_STATUS(pCxt);
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt) && NULL != pFill) {
SFillNode* pFillClause = (SFillNode*)pFill;
nodesDestroyNode(pFillClause->pWStartTs);
pFillClause->pWStartTs = createPrimaryKeyCol(pCxt);
((SSelectStmt*)pStmt)->pFill = (SNode*)pFillClause;
}
return pStmt;
}
@ -878,6 +912,7 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
pOptions->watermark1 = TSDB_DEFAULT_ROLLUP_WATERMARK;
pOptions->watermark2 = TSDB_DEFAULT_ROLLUP_WATERMARK;
pOptions->ttl = TSDB_DEFAULT_TABLE_TTL;
pOptions->commentNull = true; // mark null
return (SNode*)pOptions;
}
@ -886,6 +921,7 @@ SNode* createAlterTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = (STableOptions*)nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->ttl = -1;
pOptions->commentNull = true; // mark null
return (SNode*)pOptions;
}
@ -894,6 +930,7 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType
switch (type) {
case TABLE_OPTION_COMMENT:
if (checkComment(pCxt, (SToken*)pVal, true)) {
((STableOptions*)pOptions)->commentNull = false;
COPY_STRING_FORM_STR_TOKEN(((STableOptions*)pOptions)->comment, (SToken*)pVal);
}
break;
@ -906,9 +943,15 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType
case TABLE_OPTION_ROLLUP:
((STableOptions*)pOptions)->pRollupFuncs = pVal;
break;
case TABLE_OPTION_TTL:
((STableOptions*)pOptions)->ttl = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
case TABLE_OPTION_TTL: {
int64_t ttl = taosStr2Int64(((SToken*)pVal)->z, NULL, 10);
if (ttl > INT32_MAX) {
ttl = INT32_MAX;
}
// ttl can not be smaller than 0, because there is a limitation in sql.y (TTL NK_INTEGER)
((STableOptions*)pOptions)->ttl = ttl;
break;
}
case TABLE_OPTION_SMA:
((STableOptions*)pOptions)->pSma = pVal;
break;
@ -971,9 +1014,9 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN
pStmt->ignoreExists = ignoreExists;
pStmt->pSpecificTags = pSpecificTags;
pStmt->pValsOfTags = pValsOfTags;
pStmt->pOptions = (STableOptions*)pOptions;
nodesDestroyNode(pRealTable);
nodesDestroyNode(pUseRealTable);
nodesDestroyNode(pOptions);
return (SNode*)pStmt;
}

1
source/libs/parser/src/parInsert.c
View File

@ -803,6 +803,7 @@ static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTa
pTbReq->name = strdup(tname);
pTbReq->ctb.suid = suid;
pTbReq->ctb.pTag = (uint8_t*)pTag;
pTbReq->commentLen = -1;
return;
}

3
source/libs/parser/src/parTokenizer.c
View File

@ -81,6 +81,7 @@ static SKeyword keywordTable[] = {
{"DURATION", TK_DURATION},
{"EXISTS", TK_EXISTS},
{"EXPLAIN", TK_EXPLAIN},
{"EVERY", TK_EVERY},
{"FILL", TK_FILL},
{"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT},
@ -152,6 +153,7 @@ static SKeyword keywordTable[] = {
{"QTIME", TK_QTIME},
{"QUERIES", TK_QUERIES},
{"QUERY", TK_QUERY},
{"RANGE", TK_RANGE},
{"RATIO", TK_RATIO},
{"READ", TK_READ},
{"REDISTRIBUTE", TK_REDISTRIBUTE},
@ -258,7 +260,6 @@ static SKeyword keywordTable[] = {
// {"LP", TK_LP},
// {"RP", TK_RP},
// {"COMMA", TK_COMMA},
// {"EVERY", TK_EVERY},
// {"VARIABLE", TK_VARIABLE},
// {"UPDATE", TK_UPDATE},
// {"CHANGE", TK_CHANGE},

152
source/libs/parser/src/parTranslater.c
View File

@ -1157,6 +1157,7 @@ static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) {
pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId);
pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType);
pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType);
pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType);
}
}
@ -1931,26 +1932,33 @@ static int32_t getFillTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWin
return code;
}
static int32_t checkFill(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
SFillNode* pFill = (SFillNode*)pInterval->pFill;
static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* pInterval) {
if (FILL_MODE_NONE == pFill->mode) {
return TSDB_CODE_SUCCESS;
}
if (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) ||
TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE);
}
int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey);
int64_t intervalRange = 0;
SValueNode* pInter = (SValueNode*)pInterval->pInterval;
if (TIME_IS_VAR_DURATION(pInter->unit)) {
// interp FILL clause
if (NULL == pInterval) {
return TSDB_CODE_SUCCESS;
}
int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey);
int64_t intervalRange = 0;
if (TIME_IS_VAR_DURATION(pInterval->unit)) {
int64_t f = 1;
if (pInter->unit == 'n') {
if (pInterval->unit == 'n') {
f = 30L * MILLISECOND_PER_DAY;
} else if (pInter->unit == 'y') {
} else if (pInterval->unit == 'y') {
f = 365L * MILLISECOND_PER_DAY;
}
intervalRange = pInter->datum.i * f;
intervalRange = pInterval->datum.i * f;
} else {
intervalRange = pInter->datum.i;
intervalRange = pInterval->datum.i;
}
if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE);
@ -1966,7 +1974,7 @@ static int32_t translateFill(STranslateContext* pCxt, SNode* pWhere, SIntervalWi
int32_t code = getFillTimeRange(pCxt, pWhere, &(((SFillNode*)pInterval->pFill)->timeRange));
if (TSDB_CODE_SUCCESS == code) {
code = checkFill(pCxt, pInterval);
code = checkFill(pCxt, (SFillNode*)pInterval->pFill, (SValueNode*)pInterval->pInterval);
}
return code;
}
@ -2108,6 +2116,64 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
return code;
}
static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) {
SFillNode* pFill = (SFillNode*)nodesMakeNode(QUERY_NODE_FILL);
if (NULL == pFill) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pFill->mode = FILL_MODE_NONE;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
nodesDestroyNode((SNode*)pFill);
return TSDB_CODE_OUT_OF_MEMORY;
}
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME);
pFill->pWStartTs = (SNode*)pCol;
*pOutput = (SNode*)pFill;
return TSDB_CODE_SUCCESS;
}
static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pSelect->pFill) {
code = createDefaultFillNode(pCxt, &pSelect->pFill);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateExpr(pCxt, &pSelect->pFill);
}
if (TSDB_CODE_SUCCESS == code) {
code = getFillTimeRange(pCxt, pSelect->pRange, &(((SFillNode*)pSelect->pFill)->timeRange));
}
if (TSDB_CODE_SUCCESS == code) {
code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery);
}
return code;
}
static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasInterpFunc) {
if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE);
}
return TSDB_CODE_SUCCESS;
}
int32_t code = translateExpr(pCxt, &pSelect->pRange);
if (TSDB_CODE_SUCCESS == code) {
code = translateExpr(pCxt, &pSelect->pEvery);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateInterpFill(pCxt, pSelect);
}
return code;
}
static int32_t translatePartitionBy(STranslateContext* pCxt, SNodeList* pPartitionByList) {
pCxt->currClause = SQL_CLAUSE_PARTITION_BY;
return translateExprList(pCxt, pPartitionByList);
@ -2377,6 +2443,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (TSDB_CODE_SUCCESS == code) {
code = checkLimit(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateInterp(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteUniqueStmt(pCxt, pSelect);
}
@ -3147,9 +3216,6 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
if (TSDB_CODE_SUCCESS == code) {
code = checkTableRollupOption(pCxt, pStmt->pOptions->pRollupFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "ttl", pStmt->pOptions->ttl, TSDB_MIN_TABLE_TTL, INT32_MAX);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTableSmaOption(pCxt, pStmt);
}
@ -3390,17 +3456,19 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm
pReq->delay2 = pStmt->pOptions->maxDelay2;
pReq->watermark1 = pStmt->pOptions->watermark1;
pReq->watermark2 = pStmt->pOptions->watermark2;
pReq->ttl = pStmt->pOptions->ttl;
// pReq->ttl = pStmt->pOptions->ttl;
columnDefNodeToField(pStmt->pCols, &pReq->pColumns);
columnDefNodeToField(pStmt->pTags, &pReq->pTags);
pReq->numOfColumns = LIST_LENGTH(pStmt->pCols);
pReq->numOfTags = LIST_LENGTH(pStmt->pTags);
if ('\0' != pStmt->pOptions->comment[0]) {
if (pStmt->pOptions->commentNull == false) {
pReq->comment = strdup(pStmt->pOptions->comment);
if (NULL == pReq->comment) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReq->commentLen = strlen(pStmt->pOptions->comment) + 1;
pReq->commentLen = strlen(pStmt->pOptions->comment);
} else {
pReq->commentLen = -1;
}
SName tableName;
@ -3452,14 +3520,17 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
pAlterReq->alterType = pStmt->alterType;
if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) {
pAlterReq->ttl = pStmt->pOptions->ttl;
if ('\0' != pStmt->pOptions->comment[0]) {
// pAlterReq->ttl = pStmt->pOptions->ttl;
if (pStmt->pOptions->commentNull == false) {
pAlterReq->comment = strdup(pStmt->pOptions->comment);
if (NULL == pAlterReq->comment) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pAlterReq->commentLen = strlen(pStmt->pOptions->comment) + 1;
pAlterReq->commentLen = strlen(pStmt->pOptions->comment);
} else {
pAlterReq->commentLen = -1;
}
return TSDB_CODE_SUCCESS;
}
@ -4715,6 +4786,16 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
SVCreateTbReq req = {0};
req.type = TD_NORMAL_TABLE;
req.name = strdup(pStmt->tableName);
req.ttl = pStmt->pOptions->ttl;
if (pStmt->pOptions->commentNull == false) {
req.comment = strdup(pStmt->pOptions->comment);
if (NULL == req.comment) {
return TSDB_CODE_OUT_OF_MEMORY;
}
req.commentLen = strlen(pStmt->pOptions->comment);
} else {
req.commentLen = -1;
}
req.ntb.schemaRow.nCols = LIST_LENGTH(pStmt->pCols);
req.ntb.schemaRow.version = 1;
req.ntb.schemaRow.pSchema = taosMemoryCalloc(req.ntb.schemaRow.nCols, sizeof(SSchema));
@ -4865,6 +4946,13 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S
struct SVCreateTbReq req = {0};
req.type = TD_CHILD_TABLE;
req.name = strdup(pStmt->tableName);
req.ttl = pStmt->pOptions->ttl;
if (pStmt->pOptions->commentNull == false) {
req.comment = strdup(pStmt->pOptions->comment);
req.commentLen = strlen(pStmt->pOptions->comment);
} else {
req.commentLen = -1;
}
req.ctb.suid = suid;
req.ctb.pTag = (uint8_t*)pTag;
if (pStmt->ignoreExists) {
@ -5436,18 +5524,19 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p
int32_t code = TSDB_CODE_SUCCESS;
if (-1 != pStmt->pOptions->ttl) {
code = checkRangeOption(pCxt, "ttl", pStmt->pOptions->ttl, TSDB_MIN_TABLE_TTL, INT32_MAX);
if (TSDB_CODE_SUCCESS == code) {
pReq->updateTTL = true;
pReq->newTTL = pStmt->pOptions->ttl;
}
pReq->updateTTL = true;
pReq->newTTL = pStmt->pOptions->ttl;
}
if (TSDB_CODE_SUCCESS == code && '\0' != pStmt->pOptions->comment[0]) {
pReq->updateComment = true;
pReq->newComment = strdup(pStmt->pOptions->comment);
if (NULL == pReq->newComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
if (TSDB_CODE_SUCCESS == code) {
if (pStmt->pOptions->commentNull == false) {
pReq->newComment = strdup(pStmt->pOptions->comment);
if (NULL == pReq->newComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
pReq->newCommentLen = strlen(pReq->newComment);
} else {
pReq->newCommentLen = -1;
}
}
@ -5566,6 +5655,9 @@ static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* p
pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
}
if (pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_OPTIONS && -1 != pStmt->pOptions->ttl) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
}
return TSDB_CODE_SUCCESS;
} else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);

2
source/libs/parser/src/parUtil.c
View File

@ -196,6 +196,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "%s function does not supportted in group query";
case TSDB_CODE_PAR_INVALID_TABLE_OPTION:
return "Invalid option %s";
case TSDB_CODE_PAR_INVALID_INTERP_CLAUSE:
return "Invalid usage of RANGE clause, EVERY clause or FILL clause";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:

3684
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

15
source/libs/parser/test/parInitialATest.cpp
View File

@ -121,10 +121,10 @@ TEST_F(ParserInitialATest, alterSTable) {
int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname);
expect.name[len] = '\0';
expect.alterType = alterType;
expect.ttl = ttl;
// expect.ttl = ttl;
if (nullptr != pComment) {
expect.comment = strdup(pComment);
expect.commentLen = strlen(pComment) + 1;
expect.commentLen = strlen(pComment);
}
expect.numOfFields = numOfFields;
@ -180,9 +180,9 @@ TEST_F(ParserInitialATest, alterSTable) {
tFreeSMAltertbReq(&req);
});
setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
run("ALTER TABLE st1 TTL 10");
clearAlterStbReq();
// setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
// run("ALTER TABLE st1 TTL 10");
// clearAlterStbReq();
setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test");
run("ALTER TABLE st1 COMMENT 'test'");
@ -289,7 +289,7 @@ TEST_F(ParserInitialATest, alterTable) {
expect.newTTL = ttl;
}
if (nullptr != pComment) {
expect.updateComment = true;
expect.newCommentLen = strlen(pComment);
expect.newComment = pComment;
}
};
@ -328,9 +328,10 @@ TEST_F(ParserInitialATest, alterTable) {
ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0);
ASSERT_EQ(req.updateTTL, expect.updateTTL);
ASSERT_EQ(req.newTTL, expect.newTTL);
ASSERT_EQ(req.updateComment, expect.updateComment);
if (nullptr != expect.newComment) {
ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment));
ASSERT_EQ(req.newCommentLen, strlen(req.newComment));
ASSERT_EQ(expect.newCommentLen, strlen(expect.newComment));
}
tDecoderClear(&coder);

6
source/libs/parser/test/parInitialCTest.cpp
View File

@ -370,10 +370,10 @@ TEST_F(ParserInitialCTest, createStable) {
expect.delay2 = delay2;
expect.watermark1 = watermark1;
expect.watermark2 = watermark2;
expect.ttl = ttl;
// expect.ttl = ttl;
if (nullptr != pComment) {
expect.comment = strdup(pComment);
expect.commentLen = strlen(pComment) + 1;
expect.commentLen = strlen(pComment);
}
};
@ -414,7 +414,7 @@ TEST_F(ParserInitialCTest, createStable) {
ASSERT_EQ(req.ttl, expect.ttl);
ASSERT_EQ(req.numOfColumns, expect.numOfColumns);
ASSERT_EQ(req.numOfTags, expect.numOfTags);
ASSERT_EQ(req.commentLen, expect.commentLen);
// ASSERT_EQ(req.commentLen, expect.commentLen);
ASSERT_EQ(req.ast1Len, expect.ast1Len);
ASSERT_EQ(req.ast2Len, expect.ast2Len);

16
source/libs/parser/test/parSelectTest.cpp
View File

@ -256,6 +256,22 @@ TEST_F(ParserSelectTest, intervalSemanticCheck) {
run("SELECT _WSTARTTS, _WENDTS, _WDURATION, sum(c1) FROM t1", TSDB_CODE_PAR_INVALID_WINDOW_PC);
}
TEST_F(ParserSelectTest, interp) {
useDb("root", "test");
run("SELECT INTERP(c1) FROM t1");
run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00')");
run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') FILL(LINEAR)");
run("SELECT INTERP(c1) FROM t1 EVERY(5s)");
run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s)");
run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
}
TEST_F(ParserSelectTest, subquery) {
useDb("root", "test");

4
source/libs/planner/inc/planInt.h
View File

@ -23,10 +23,6 @@ extern "C" {
#include "planner.h"
#include "taoserror.h"
#define QUERY_POLICY_VNODE 1
#define QUERY_POLICY_HYBRID 2
#define QUERY_POLICY_QNODE 3
#define planFatal(param, ...) qFatal("PLAN: " param, __VA_ARGS__)
#define planError(param, ...) qError("PLAN: " param, __VA_ARGS__)
#define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__)

67
source/libs/planner/src/planLogicCreater.c
View File

@ -91,7 +91,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
nodesWalkExprs(pExprs, doNameExpr, NULL);
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs};
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
@ -266,7 +266,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
}
pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType);
@ -425,7 +425,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) {
@ -434,7 +434,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) {
@ -473,14 +473,15 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pVectorFuncs);
// indefinite rows functions and _select_values functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprForSelect(pIdfRowsFunc->pVectorFuncs, pSelect, SQL_CLAUSE_SELECT);
code = rewriteExprsForSelect(pIdfRowsFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT);
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExprs(pIdfRowsFunc->pVectorFuncs, &pIdfRowsFunc->node.pTargets);
code = createColumnByRewriteExprs(pIdfRowsFunc->pFuncs, &pIdfRowsFunc->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
@ -492,6 +493,50 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
return code;
}
static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (!pSelect->hasInterpFunc) {
return TSDB_CODE_SUCCESS;
}
SInterpFuncLogicNode* pInterpFunc = (SInterpFuncLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_INTERP_FUNC);
if (NULL == pInterpFunc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT);
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pFill) {
SFillNode* pFill = (SFillNode*)pSelect->pFill;
pInterpFunc->timeRange = pFill->timeRange;
pInterpFunc->fillMode = pFill->mode;
pInterpFunc->pTimeSeries = nodesCloneNode(pFill->pWStartTs);
pInterpFunc->pFillValues = nodesCloneNode(pFill->pValues);
if (NULL == pInterpFunc->pTimeSeries || (NULL != pFill->pValues && NULL == pInterpFunc->pFillValues)) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pEvery) {
pInterpFunc->interval = ((SValueNode*)pSelect->pEvery)->datum.i;
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExprs(pInterpFunc->pFuncs, &pInterpFunc->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pInterpFunc;
} else {
nodesDestroyNode((SNode*)pInterpFunc);
}
return code;
}
static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow,
SLogicNode** pLogicNode) {
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs);
@ -506,7 +551,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
}
if (TSDB_CODE_SUCCESS == code) {
@ -559,6 +604,7 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW
nodesDestroyNode((SNode*)pWindow);
return TSDB_CODE_OUT_OF_MEMORY;
}
pWindow->pTsEnd = nodesCloneNode((SNode*)pSession->pCol);
return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode);
}
@ -777,7 +823,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
}
// set the output
@ -812,6 +858,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele
if (TSDB_CODE_SUCCESS == code) {
code = createSelectRootLogicNode(pCxt, pSelect, createIndefRowsFuncLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createSelectRootLogicNode(pCxt, pSelect, createInterpFuncLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createSelectRootLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot);
}

76
source/libs/planner/src/planPhysiCreater.c
View File

@ -496,10 +496,6 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable;
}
if (pCxt->pExecNodeList) {
SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
taosArrayPush(pCxt->pExecNodeList, &node);
}
tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
pTableScan->dataRequired = pScanLogicNode->dataRequired;
pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs);
@ -535,12 +531,9 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES) ||
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)) {
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
taosArrayPush(pCxt->pExecNodeList, &node);
} else {
SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
taosArrayPush(pCxt->pExecNodeList, &node);
}
SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
taosArrayPush(pCxt->pExecNodeList, &node);
pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet;
tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
@ -825,8 +818,8 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList*
}
SNodeList* pPrecalcExprs = NULL;
SNodeList* pVectorFuncs = NULL;
int32_t code = rewritePrecalcExprs(pCxt, pFuncLogicNode->pVectorFuncs, &pPrecalcExprs, &pVectorFuncs);
SNodeList* pFuncs = NULL;
int32_t code = rewritePrecalcExprs(pCxt, pFuncLogicNode->pFuncs, &pPrecalcExprs, &pFuncs);
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
// push down expression to pOutputDataBlockDesc of child node
@ -837,10 +830,10 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList*
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pVectorFuncs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pVectorFuncs, &pIdfRowsFunc->pVectorFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pIdfRowsFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pIdfRowsFunc->pVectorFuncs, pIdfRowsFunc->node.pOutputDataBlockDesc);
code = addDataBlockSlots(pCxt, pIdfRowsFunc->pFuncs, pIdfRowsFunc->node.pOutputDataBlockDesc);
}
}
@ -853,6 +846,54 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList*
return code;
}
static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SInterpFuncLogicNode* pFuncLogicNode, SPhysiNode** pPhyNode) {
SInterpFuncPhysiNode* pInterpFunc =
(SInterpFuncPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFuncLogicNode, QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC);
if (NULL == pInterpFunc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeList* pPrecalcExprs = NULL;
SNodeList* pFuncs = NULL;
int32_t code = rewritePrecalcExprs(pCxt, pFuncLogicNode->pFuncs, &pPrecalcExprs, &pFuncs);
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
// push down expression to pOutputDataBlockDesc of child node
if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pInterpFunc->pExprs);
if (TSDB_CODE_SUCCESS == code) {
code = pushdownDataBlockSlots(pCxt, pInterpFunc->pExprs, pChildTupe);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pInterpFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pInterpFunc->pFuncs, pInterpFunc->node.pOutputDataBlockDesc);
}
}
if (TSDB_CODE_SUCCESS == code) {
pInterpFunc->timeRange = pFuncLogicNode->timeRange;
pInterpFunc->interval = pFuncLogicNode->interval;
pInterpFunc->fillMode = pFuncLogicNode->fillMode;
pInterpFunc->pFillValues = nodesCloneNode(pFuncLogicNode->pFillValues);
pInterpFunc->pTimeSeries = nodesCloneNode(pFuncLogicNode->pTimeSeries);
if (NULL == pInterpFunc->pTimeSeries || (NULL != pFuncLogicNode->pFillValues && NULL == pInterpFunc->pFillValues)) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pInterpFunc;
} else {
nodesDestroyNode((SNode*)pInterpFunc);
}
return code;
}
static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) {
SProjectPhysiNode* pProject =
@ -961,6 +1002,9 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList*
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pTspk, &pWindow->pTspk);
}
if (TSDB_CODE_SUCCESS == code && pWindowLogicNode->pTsEnd) {
code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pTsEnd, &pWindow->pTsEnd);
}
if (TSDB_CODE_SUCCESS == code && NULL != pFuncs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pWindow->pFuncs);
@ -1295,6 +1339,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode
return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
return createIndefRowsFuncPhysiNode(pCxt, pChildren, (SIndefRowsFuncLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return createInterpFuncPhysiNode(pCxt, pChildren, (SInterpFuncLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_MERGE:
return createMergePhysiNode(pCxt, (SMergeLogicNode*)pLogicNode, pPhyNode);
default:
@ -1385,8 +1431,6 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
static int32_t buildInsertSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
pSubplan->msgType = pModify->msgType;
pSubplan->execNode.epSet = pModify->pVgDataBlocks->vg.epSet;
SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
taosArrayPush(pCxt->pExecNodeList, &node);
return createDataInserter(pCxt, pModify->pVgDataBlocks, &pSubplan->pDataSink);
}

44
source/libs/planner/src/planSpliter.c
View File

@ -176,7 +176,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
case QUERY_NODE_LOGIC_PLAN_WINDOW: {
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
if (WINDOW_TYPE_INTERVAL != pWindow->winType) {
if (WINDOW_TYPE_STATE == pWindow->winType || (!streamQuery && WINDOW_TYPE_SESSION == pWindow->winType) ) {
return false;
}
return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
@ -257,6 +257,34 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) {
return code;
}
static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) {
int32_t index = 0;
SNode* pFunc = NULL;
FOREACH(pFunc, pWin->pFuncs) {
if (FUNCTION_TYPE_WENDTS == ((SFunctionNode*)pFunc)->funcType) {
*pIndex = index;
return TSDB_CODE_SUCCESS;
}
++index;
}
SFunctionNode* pWEnd = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pWEnd) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pWEnd->functionName, "_wendts");
snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd);
int32_t code = fmGetFuncInfo(pWEnd, NULL, 0);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd);
}
*pIndex = index;
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExpr(nodesListGetNode(pWin->pFuncs, index), &pWin->node.pTargets);
}
return code;
}
static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogicNode** pPartWindow) {
SNodeList* pFunc = pMergeWindow->pFuncs;
pMergeWindow->pFuncs = NULL;
@ -425,8 +453,18 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
SLogicNode* pPartWindow = NULL;
int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow);
if (TSDB_CODE_SUCCESS == code) {
((SWindowLogicNode*)pPartWindow)->windowAlgo = SESSION_ALGO_STREAM_SEMI;
((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = SESSION_ALGO_STREAM_FINAL;
SWindowLogicNode* pPartWin = (SWindowLogicNode*)pPartWindow;
SWindowLogicNode* pMergeWin = (SWindowLogicNode*)pInfo->pSplitNode;
pPartWin->windowAlgo = SESSION_ALGO_STREAM_SEMI;
pMergeWin->windowAlgo = SESSION_ALGO_STREAM_FINAL;
int32_t index = 0;
int32_t code = stbSplAppendWEnd(pPartWin, &index);
if (TSDB_CODE_SUCCESS == code) {
pMergeWin->pTsEnd = nodesCloneNode(nodesListGetNode(pPartWin->node.pTargets, index));
if (NULL == pMergeWin->pTsEnd) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
}
if (TSDB_CODE_SUCCESS == code) {

10
source/libs/planner/test/planBasicTest.cpp
View File

@ -52,6 +52,8 @@ TEST_F(PlanBasicTest, func) {
run("SELECT PERCENTILE(c1, 60) FROM t1");
run("SELECT TOP(c1, 60) FROM t1");
run("SELECT TOP(c1, 60) FROM st1");
}
TEST_F(PlanBasicTest, uniqueFunc) {
@ -73,3 +75,11 @@ TEST_F(PlanBasicTest, tailFunc) {
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10");
}
TEST_F(PlanBasicTest, interpFunc) {
useDb("root", "test");
run("SELECT INTERP(c1) FROM t1");
run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
}

1
source/libs/qworker/inc/qwMsg.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "qwInt.h"
#include "dataSinkMgt.h"
int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF);
int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain, const char* sql);
int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg);

20
source/libs/qworker/src/qwMsg.c
View File

@ -283,6 +283,26 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
if (NULL == qWorkerMgmt || NULL == pMsg) {
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SSubQueryMsg *msg = pMsg->pCont;
SQWorker * mgmt = (SQWorker *)qWorkerMgmt;
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t tId = msg->taskId;
int64_t rId = msg->refId;
QW_SCH_TASK_DLOG("Abort prerocessQuery start, handle:%p", pMsg->info.handle);
qwAbortPrerocessQuery(QW_FPARAMS());
QW_SCH_TASK_DLOG("Abort prerocessQuery end, handle:%p", pMsg->info.handle);
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);

7
source/libs/qworker/src/qworker.c
View File

@ -482,6 +482,13 @@ _return:
QW_RET(code);
}
int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) {
QW_ERR_RET(qwDropTask(QW_FPARAMS()));
QW_RET(TSDB_CODE_SUCCESS);
}
int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0;
bool queryRsped = false;

4
source/libs/scheduler/src/schJob.c
View File

@ -652,7 +652,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
if (addNum <= 0) {
SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum);
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
SCH_ERR_RET(TSDB_CODE_TSC_NO_EXEC_NODE);
}
return TSDB_CODE_SUCCESS;
@ -677,7 +677,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
SCH_TASK_DLOG("use execNode in plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
return TSDB_CODE_SUCCESS;
}

3
source/libs/sync/inc/syncInt.h
View File

@ -238,6 +238,9 @@ int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg);
bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId);
SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId);
int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta);
int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct SSnapshotMeta* sMeta);
void syncStartNormal(int64_t rid);
void syncStartStandBy(int64_t rid);

4
source/libs/sync/src/syncAppendEntries.c
View File

@ -497,7 +497,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
do {
SyncIndex myLastIndex = syncNodeGetLastIndex(ths);
SSnapshot snapshot;
ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) &&
syncNodeHasSnapshot(ths);
@ -710,7 +710,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) {
// advance commit index to sanpshot first
SSnapshot snapshot;
ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
if (snapshot.lastApplyIndex >= 0 && snapshot.lastApplyIndex > ths->commitIndex) {
SyncIndex commitBegin = ths->commitIndex;
SyncIndex commitEnd = snapshot.lastApplyIndex;

6
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -123,7 +123,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex);
if (gRaftDetailLog) {
SSnapshot snapshot;
ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
}
@ -175,7 +175,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
ASSERT(pSender != NULL);
bool hasSnapshot = syncNodeHasSnapshot(ths);
SSnapshot snapshot;
ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
// start sending snapshot first time
// start here, stop by receiver
@ -209,7 +209,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex);
if (gRaftDetailLog) {
SSnapshot snapshot;
ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
}

2
source/libs/sync/src/syncCommit.c
View File

@ -50,7 +50,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
// advance commit index to sanpshot first
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
if (snapshot.lastApplyIndex > 0 && snapshot.lastApplyIndex > pSyncNode->commitIndex) {
SyncIndex commitBegin = pSyncNode->commitIndex;
SyncIndex commitEnd = snapshot.lastApplyIndex;

24
source/libs/sync/src/syncMain.c
View File

@ -809,9 +809,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
pSyncNode->restoreFinish = false;
// pSyncNode->pSnapshot = NULL;
// if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
// if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
// pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot));
// pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot);
// pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, pSyncNode->pSnapshot);
// }
// tsem_init(&(pSyncNode->restoreSem), 0, 0);
@ -1658,8 +1658,8 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) {
bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
bool ret = false;
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN) {
ret = true;
}
@ -1669,19 +1669,19 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) {
ASSERT(syncNodeHasSnapshot(pSyncNode));
ASSERT(pSyncNode->pFsm->FpGetSnapshot != NULL);
ASSERT(pSyncNode->pFsm->FpGetSnapshotInfo != NULL);
ASSERT(index >= SYNC_INDEX_BEGIN);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
bool b = (index <= snapshot.lastApplyIndex);
return b;
}
SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) {
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
}
SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
@ -1694,8 +1694,8 @@ SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) {
if (syncNodeHasSnapshot(pSyncNode)) {
// has snapshot
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
}
SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
@ -1747,8 +1747,8 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
if (syncNodeHasSnapshot(pSyncNode)) {
// has snapshot
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
}
if (index > snapshot.lastApplyIndex + 1) {

2
source/libs/sync/src/syncReplication.c
View File

@ -124,7 +124,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) {
logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore);
if (gRaftDetailLog) {
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
sTrace("begin append entries peers, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
}

6
source/libs/sync/src/syncSnapshot.c
View File

@ -45,7 +45,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
pSender->replicaIndex = replicaIndex;
pSender->term = pSyncNode->pRaftStore->currentTerm;
pSender->privateTerm = taosGetTimestampMs() + 100;
pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot));
pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
pSender->finish = false;
} else {
sError("snapshotSenderCreate cannot create sender");
@ -84,7 +84,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) {
pSender->blockLen = 0;
// get current snapshot info
pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot));
pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
sTrace("snapshotSenderStart lastApplyIndex:%ld, lastApplyTerm:%lu, lastConfigIndex:%ld",
pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex);
@ -558,7 +558,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
}
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
do {
char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver finish");

6
source/libs/sync/test/syncConfigChangeSnapshotTest.cpp
View File

@ -36,9 +36,9 @@ void cleanup() { walCleanUp(); }
void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
SyncIndex beginIndex = SYNC_INDEX_INVALID;
if (pFsm->FpGetSnapshot != NULL) {
if (pFsm->FpGetSnapshotInfo != NULL) {
SSnapshot snapshot;
pFsm->FpGetSnapshot(pFsm, &snapshot);
pFsm->FpGetSnapshotInfo(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex;
}
@ -159,7 +159,7 @@ SSyncFSM* createFsm() {
pFsm->FpPreCommitCb = PreCommitCb;
pFsm->FpRollBackCb = RollBackCb;
pFsm->FpGetSnapshot = GetSnapshotCb;
pFsm->FpGetSnapshotInfo = GetSnapshotCb;
pFsm->FpRestoreFinishCb = RestoreFinishCb;
pFsm->FpSnapshotStartRead = SnapshotStartRead;
pFsm->FpSnapshotStopRead = SnapshotStopRead;

6
source/libs/sync/test/syncConfigChangeTest.cpp
View File

@ -35,9 +35,9 @@ void cleanup() { walCleanUp(); }
void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
SyncIndex beginIndex = SYNC_INDEX_INVALID;
if (pFsm->FpGetSnapshot != NULL) {
if (pFsm->FpGetSnapshotInfo != NULL) {
SSnapshot snapshot;
pFsm->FpGetSnapshot(pFsm, &snapshot);
pFsm->FpGetSnapshotInfo(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex;
}
@ -90,7 +90,7 @@ SSyncFSM* createFsm() {
pFsm->FpPreCommitCb = PreCommitCb;
pFsm->FpRollBackCb = RollBackCb;
pFsm->FpGetSnapshot = GetSnapshotCb;
pFsm->FpGetSnapshotInfo = GetSnapshotCb;
pFsm->FpRestoreFinishCb = RestoreFinishCb;
pFsm->FpReConfigCb = ReConfigCb;

2
source/libs/sync/test/syncRaftLogTest2.cpp
View File

@ -54,7 +54,7 @@ void init() {
pSyncNode->pWal = pWal;
pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
pSyncNode->pFsm->FpGetSnapshot = GetSnapshotCb;
pSyncNode->pFsm->FpGetSnapshotInfo = GetSnapshotCb;
}
void cleanup() {

12
source/libs/sync/test/syncRaftLogTest3.cpp
View File

@ -54,7 +54,7 @@ void init() {
pSyncNode->pWal = pWal;
pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
pSyncNode->pFsm->FpGetSnapshot = GetSnapshotCb;
pSyncNode->pFsm->FpGetSnapshotInfo = GetSnapshotCb;
}
void cleanup() {
@ -80,7 +80,7 @@ void test1() {
bool hasSnapshot = syncNodeHasSnapshot(pSyncNode);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode);
SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode);
@ -146,7 +146,7 @@ void test2() {
bool hasSnapshot = syncNodeHasSnapshot(pSyncNode);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode);
SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode);
@ -203,7 +203,7 @@ void test3() {
bool hasSnapshot = syncNodeHasSnapshot(pSyncNode);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode);
SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode);
@ -268,7 +268,7 @@ void test4() {
bool hasSnapshot = syncNodeHasSnapshot(pSyncNode);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode);
SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode);
@ -335,7 +335,7 @@ void test5() {
bool hasSnapshot = syncNodeHasSnapshot(pSyncNode);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode);
SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode);

6
source/libs/sync/test/syncReplicateTest.cpp
View File

@ -32,9 +32,9 @@ void cleanup() { walCleanUp(); }
void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
SyncIndex beginIndex = SYNC_INDEX_INVALID;
if (pFsm->FpGetSnapshot != NULL) {
if (pFsm->FpGetSnapshotInfo != NULL) {
SSnapshot snapshot;
pFsm->FpGetSnapshot(pFsm, &snapshot);
pFsm->FpGetSnapshotInfo(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex;
}
@ -75,7 +75,7 @@ SSyncFSM* createFsm() {
pFsm->FpCommitCb = CommitCb;
pFsm->FpPreCommitCb = PreCommitCb;
pFsm->FpRollBackCb = RollBackCb;
pFsm->FpGetSnapshot = GetSnapshotCb;
pFsm->FpGetSnapshotInfo = GetSnapshotCb;
return pFsm;
}

2
source/libs/sync/test/syncSnapshotSenderTest.cpp
View File

@ -40,7 +40,7 @@ SSyncSnapshotSender* createSender() {
pSyncNode->pFsm->FpSnapshotStartRead = SnapshotStartRead;
pSyncNode->pFsm->FpSnapshotStopRead = SnapshotStopRead;
pSyncNode->pFsm->FpSnapshotDoRead = SnapshotDoRead;
pSyncNode->pFsm->FpGetSnapshot = GetSnapshot;
pSyncNode->pFsm->FpGetSnapshotInfo = GetSnapshot;
SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, 2);
pSender->start = true;

6
source/libs/sync/test/syncSnapshotTest.cpp
View File

@ -35,9 +35,9 @@ const char *pWalDir = "./syncSnapshotTest_wal";
void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
SyncIndex beginIndex = SYNC_INDEX_INVALID;
if (pFsm->FpGetSnapshot != NULL) {
if (pFsm->FpGetSnapshotInfo != NULL) {
SSnapshot snapshot;
pFsm->FpGetSnapshot(pFsm, &snapshot);
pFsm->FpGetSnapshotInfo(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex;
}
@ -79,7 +79,7 @@ void initFsm() {
pFsm->FpCommitCb = CommitCb;
pFsm->FpPreCommitCb = PreCommitCb;
pFsm->FpRollBackCb = RollBackCb;
pFsm->FpGetSnapshot = GetSnapshotCb;
pFsm->FpGetSnapshotInfo = GetSnapshotCb;
}
SSyncNode *syncNodeInit() {

2
source/libs/sync/test/syncTestTool.cpp
View File

@ -172,7 +172,7 @@ SSyncFSM* createFsm() {
pFsm->FpRollBackCb = RollBackCb;
pFsm->FpReConfigCb = ReConfigCb;
pFsm->FpGetSnapshot = GetSnapshotCb;
pFsm->FpGetSnapshotInfo = GetSnapshotCb;
pFsm->FpRestoreFinishCb = RestoreFinishCb;
pFsm->FpSnapshotStartRead = SnapshotStartRead;

1
source/libs/tdb/inc/tdb.h
View File

@ -58,6 +58,7 @@ int32_t tdbTbcMoveToPrev(TBC *pTbc);
int32_t tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
int32_t tdbTbcDelete(TBC *pTbc);
int32_t tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int32_t tdbTbcPrev(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int32_t tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert);
// TXN

46
source/libs/tdb/src/db/tdbBtree.c
View File

@ -1245,6 +1245,52 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return 0;
}
int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
SCell *pCell;
SCellDecoder cd;
void *pKey, *pVal;
int ret;
// current cursor points to an invalid position
if (pBtc->idx < 0) {
return -1;
}
pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx);
tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd);
pKey = tdbRealloc(*ppKey, cd.kLen);
if (pKey == NULL) {
return -1;
}
*ppKey = pKey;
*kLen = cd.kLen;
memcpy(pKey, cd.pKey, cd.kLen);
if (ppVal) {
// TODO: vLen may be zero
pVal = tdbRealloc(*ppVal, cd.vLen);
if (pVal == NULL) {
tdbFree(pKey);
return -1;
}
*ppVal = pVal;
*vLen = cd.vLen;
memcpy(pVal, cd.pVal, cd.vLen);
}
ret = tdbBtcMoveToPrev(pBtc);
if (ret < 0) {
ASSERT(0);
return -1;
}
return 0;
}
int tdbBtcMoveToNext(SBTC *pBtc) {
int nCells;
int ret;

4
source/libs/tdb/src/db/tdbTable.c
View File

@ -132,6 +132,10 @@ int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreeNext(&pTbc->btc, ppKey, kLen, ppVal, vLen);
}
int tdbTbcPrev(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreePrev(&pTbc->btc, ppKey, kLen, ppVal, vLen);
}
int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
return tdbBtcUpsert(&pTbc->btc, pKey, nKey, pData, nData, insert);
}

1
source/libs/tdb/src/inc/tdbInt.h
View File

@ -156,6 +156,7 @@ int tdbBtcMoveToLast(SBTC *pBtc);
int tdbBtcMoveToNext(SBTC *pBtc);
int tdbBtcMoveToPrev(SBTC *pBtc);
int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
int tdbBtcDelete(SBTC *pBtc);
int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert);

1
source/util/src/terror.c
View File

@ -133,6 +133,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_API_ERROR, "Stmt API usage error"
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node")
// mnode-common
TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error")

129
tests/pytest/util/common.py
View File

@ -11,13 +11,20 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import random
import string
from util.sql import tdSql
from util.dnodes import tdDnodes
import requests
import time
import socket
import taos
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
class TDCom:
def init(self, conn, logSql):
tdSql.init(conn.cursor(), logSql)
@ -170,4 +177,122 @@ class TDCom:
def close(self):
self.cursor.close()
def create_database(self,tsql, dbName='test',dropFlag=1,precision="ms", **kwargs):
if dropFlag == 1:
tsql.execute("drop database if exists %s"%(dbName))
'''
vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions
'''
sqlString = f'create database if not exists {dbName} precision "{precision}" vgroups 4'
if len(kwargs) > 0:
dbParams = ""
for param, value in kwargs.items():
dbParams += f'{param} {value} '
sqlString += f'{dbParams}'
tsql.execute(sqlString)
tdLog.debug("complete to create database %s"%(dbName))
return
def create_stable(self,tsql, dbName,stbName,columnDict,tagDict):
colSchema = ''
for i in range(columnDict['int']):
colSchema += ', c%d int'%i
tagSchema = ''
for i in range(tagDict['int']):
if i > 0:
tagSchema += ','
tagSchema += 't%d int'%i
tsql.execute("create table if not exists %s.%s (ts timestamp %s) tags(%s)"%(dbName, stbName, colSchema, tagSchema))
tdLog.debug("complete to create %s.%s" %(dbName, stbName))
return
def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict):
tsql.execute("use %s" %dbName)
tagsValues = ''
for i in range(tagDict['int']):
if i > 0:
tagsValues += ','
tagsValues += '%d'%i
pre_create = "create table"
sql = pre_create
#tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
for i in range(ctbNum):
sql += " %s_%d using %s tags(%s)"%(stbName,i,stbName,tagsValues)
if (i > 0) and (i%100 == 0):
tsql.execute(sql)
sql = pre_create
if sql != pre_create:
tsql.execute(sql)
tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
return
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0):
tdLog.debug("start to insert data ............")
tsql.execute("use %s" %dbName)
pre_insert = "insert into "
sql = pre_insert
if startTs == 0:
t = time.time()
startTs = int(round(t * 1000))
#tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
for i in range(ctbNum):
sql += " %s_%d values "%(stbName,i)
for j in range(rowsPerTbl):
sql += "(%d, %d, %d)"%(startTs + j, j, j)
if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
tsql.execute(sql)
if j < rowsPerTbl - 1:
sql = "insert into %s_%d values " %(stbName,i)
else:
sql = "insert into "
#end sql
if sql != pre_insert:
#print("insert sql:%s"%sql)
tsql.execute(sql)
tdLog.debug("insert data ............ [OK]")
return
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def getClientCfgPath(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
cfgPath = buildPath + "/../sim/psim/cfg"
tdLog.info("cfgPath: %s" % cfgPath)
return cfgPath
def newcur(self,host='localhost',port=6030,user='root',password='taosdata'):
cfgPath = self.getClientCfgPath()
con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port)
cur=con.cursor()
print(cur)
return cur
def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'):
newTdSql = TDSql()
cur = self.newcur(host=host,port=port,user=user,password=password)
newTdSql.init(cur, False)
return newTdSql
tdCom = TDCom()

1
tests/script/jenkins/basic.txt
View File

@ -73,6 +73,7 @@
./test.sh -f tsim/stream/basic1.sim
./test.sh -f tsim/stream/basic2.sim
# ./test.sh -f tsim/stream/distributeInterval0.sim
# ./test.sh -f tsim/stream/distributesession0.sim
# ./test.sh -f tsim/stream/session0.sim
# ./test.sh -f tsim/stream/session1.sim
# ./test.sh -f tsim/stream/state0.sim

2
tests/script/tsim/stable/alter_comment.sim
View File

@ -159,7 +159,7 @@ sql alter table db.stb rename tag t1 tx
print ========== alter common
sql alter table db.stb comment 'abcde' ;
sql alter table db.stb ttl 10 ;
sql_error alter table db.stb ttl 10 ;
sql show db.stables;
if $data[0][6] != abcde then

58
tests/script/tsim/stream/distributesession0.sim Normal file
View File

@ -0,0 +1,58 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
sql create dnode $hostname2 port 7200
system sh/exec.sh -n dnode2 -s start
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
sql create stream stream_t1 trigger at_once into streamtST as select _wstartts, count(*) c1, sum(a) c2 , max(b) c3 from st session(ts, 10s) ;
sleep 1000
sql insert into ts1 values(1648791211000,1,1,1) (1648791211005,1,1,1);
sql insert into ts2 values(1648791221004,1,2,3) (1648791221008,2,2,3);
sql insert into ts1 values(1648791211005,1,1,1);
sql insert into ts2 values(1648791221006,5,5,5) (1648791221007,5,5,5);
sql insert into ts2 values(1648791221008,5,5,5) (1648791221008,5,5,5)(1648791221006,5,5,5);
sql insert into ts1 values(1648791231000,1,1,1) (1648791231002,1,1,1) (1648791231006,1,1,1);
sql insert into ts1 values(1648791211000,6,6,6) (1648791231002,2,2,2);
sql insert into ts1 values(1648791211002,7,7,7);
sql insert into ts1 values(1648791211002,7,7,7) ts2 values(1648791221008,5,5,5) ;
$loop_count = 0
loop1:
sql select * from streamtST;
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 10 then
print =====data01=$data01
goto loop1
endi
if $data02 != 34 then
print =====data02=$data02
goto loop1
endi
if $data03 != 7 then
print ======$data03
return -1
endi
system sh/stop_dnodes.sh

13
tests/script/tsim/stream/session0.sim
View File

@ -231,9 +231,10 @@ sql use test3;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams3 trigger at_once watermark 1d into streamt3 as select _wstartts, min(b), a,c from t1 session(ts,10s);
sql create stream streams4 trigger at_once watermark 1d into streamt4 as select _wstartts, max(b), a,c from t1 session(ts,10s);
sql create stream streams5 trigger at_once watermark 1d into streamt5 as select _wstartts, top(b,3), a,c from t1 session(ts,10s);
sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, bottom(b,3), a,c from t1 session(ts,10s);
sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s);
# sql create stream streams5 trigger at_once watermark 1d into streamt5 as select _wstartts, top(b,3), a,c from t1 session(ts,10s);
# sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, bottom(b,3), a,c from t1 session(ts,10s);
# sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s);
sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), hyperloglog(a) from t1 session(ts,10s);
sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s);
sql insert into t1 values(1648791213001,1,1,1,1.0);
sql insert into t1 values(1648791213002,2,3,2,3.4);
@ -269,13 +270,13 @@ if $rows == 0 then
goto loop3
endi
sql select * from streamt5;
#sql select * from streamt5;
if $rows == 0 then
print ======$rows
goto loop3
# goto loop3
endi
sql select * from streamt6;
# sql select * from streamt6;
if $rows == 0 then
print ======$rows
goto loop3

2
tests/script/tsim/stream/triggerSession0.sim
View File

@ -85,7 +85,7 @@ sql insert into t2 values(1648791243003,1,2,3,1.0) (1648791243002,1,2,3,1.0) (16
sleep 500
sql select * from streamt2;
if $rows != 3 then
print ======$rows
print =====rows=$rows
return -1
endi

519
tests/system-test/2-query/Today.py
View File

@ -4,394 +4,163 @@ from util.dnodes import *
from util.log import *
from util.sql import *
from util.cases import *
import datetime
class TDTestCase:
updatecfgDict = {'rpcDebugFlag': '143'}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
self.today_date = datetime.datetime.strptime(
datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
self.time_unit = ['b','u','a','s','m','h','d','w']
self.error_param = ['1.5','abc','!@#','"abc"','today()']
self.arithmetic_operators = ['+','-','*','/']
self.relational_operator = ['<','<=','=','>=','>']
# prepare data
self.ntbname = 'ntb'
self.stbname = 'stb'
self.column_dict = {
'ts':'timestamp',
'c1':'int',
'c2':'float',
'c3':'double',
'c4':'timestamp'
}
self.tag_dict = {
't0':'int'
}
self.tbnum = 2
self.tag_values = [
f'10',
f'100'
]
self.values_list = [f'now,1,1.55,100.555555,today()',
f'now+1d,10,11.11,99.999999,now()',
f'today(),3,3.333,333.333333,now()',
f'today()-1d,10,11.11,99.999999,now()',
f'today()+1d,1,1.55,100.555555,today()']
def set_create_normaltable_sql(self, ntbname, column_dict):
column_sql = ''
for k, v in column_dict.items():
column_sql += f"{k} {v},"
create_ntb_sql = f'create table {ntbname} ({column_sql[:-1]})'
return create_ntb_sql
def set_create_stable_sql(self,stbname,column_dict,tag_dict):
column_sql = ''
tag_sql = ''
for k,v in column_dict.items():
column_sql += f"{k} {v},"
for k,v in tag_dict.items():
tag_sql += f"{k} {v},"
create_stb_sql = f'create table {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})'
return create_stb_sql
def data_check(self,column_dict={},tbname = '',values_list = [],tb_num = 1,tb = 'tb'):
for k,v in column_dict.items():
num_up = 0
num_down = 0
num_same = 0
if v.lower() == 'timestamp':
tdSql.query(f'select {k} from {tbname}')
for i in tdSql.queryResult:
if i[0] > self.today_date:
num_up += 1
elif i[0] == self.today_date:
num_same += 1
elif i[0] < self.today_date:
num_down += 1
tdSql.query(f"select today() from {tbname}")
tdSql.checkRows(len(values_list)*tb_num)
tdSql.checkData(0, 0, str(self.today_date))
tdSql.query(f"select * from {tbname} where {k}=today()")
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
elif tb == 'stb':
tdSql.checkRows(num_same)
for i in [f'{tbname}',f'db.{tbname}']:
for unit in self.time_unit:
for symbol in ['+','-']:
tdSql.query(f"select today() {symbol}1{unit} from {i}")
tdSql.checkRows(len(values_list)*tb_num)
for unit in self.error_param:
for symbol in self.arithmetic_operators:
tdSql.error(f'select today() {symbol}{unit} from {i}')
for symbol in self.arithmetic_operators:
tdSql.query(f'select now(){symbol}null from {i}')
tdSql.checkData(0,0,None)
for symbol in self.relational_operator:
tdSql.query(f'select * from {i} where {k} {symbol} today()')
if symbol == '<' :
if tb == 'tb':
tdSql.checkRows(num_down*tb_num)
elif tb == 'stb':
tdSql.checkRows(num_down)
elif symbol == '<=':
if tb == 'tb':
tdSql.checkRows((num_same+num_down)*tb_num)
elif tb == 'stb':
tdSql.checkRows(num_same+num_down)
elif symbol == '=':
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
elif tb == 'stb':
tdSql.checkRows(num_same)
elif symbol == '>=':
if tb == 'tb':
tdSql.checkRows((num_up + num_same)*tb_num)
elif tb == 'stb':
tdSql.checkRows(num_up + num_same)
elif symbol == '>':
if tb == 'tb':
tdSql.checkRows(num_up*tb_num)
elif tb == 'stb':
tdSql.checkRows(num_up)
tdSql.query(f"select today()/0 from {tbname}")
tdSql.checkRows(len(values_list)*tb_num)
tdSql.checkData(0,0,None)
tdSql.query(f"select {k} from {tbname} where {k}=today()")
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
for i in range(num_same*tb_num):
tdSql.checkData(i, 0, str(self.today_date))
elif tb == 'stb':
tdSql.checkRows(num_same)
for i in range(num_same):
tdSql.checkData(i, 0, str(self.today_date))
def today_check_ntb(self):
tdSql.prepare()
tdSql.execute(self.set_create_normaltable_sql(self.ntbname,self.column_dict))
for i in self.values_list:
tdSql.execute(
f'insert into {self.ntbname} values({i})')
self.data_check(self.column_dict,self.ntbname,self.values_list)
tdSql.execute('drop database db')
def today_check_stb_tb(self):
tdSql.prepare()
tdSql.execute(self.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
for i in range(self.tbnum):
tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})')
for j in self.values_list:
tdSql.execute(f'insert into {self.stbname}_{i} values ({j})')
# check child table
for i in range(self.tbnum):
self.data_check(self.column_dict,f'{self.stbname}_{i}',self.values_list)
# check stable
self.data_check(self.column_dict,self.stbname,self.values_list,self.tbnum,'stb')
tdSql.execute('drop database db')
def run(self): # sourcery skip: extract-duplicate-method
# for func now() , today(), timezone()
tdSql.prepare()
today_date = datetime.datetime.strptime(
datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
tdLog.printNoPrefix("==========check today() for normal table ==========")
self.today_check_ntb()
tdLog.printNoPrefix("==========check today() for stable and child table==========")
self.today_check_stb_tb()
tdLog.printNoPrefix("==========step1:create tables==========")
tdSql.execute(
'''create table if not exists ntb
(ts timestamp, c1 int, c2 float,c3 double,c4 timestamp)
'''
)
tdSql.execute(
'''create table if not exists stb
(ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int)
'''
)
tdSql.execute(
'''create table if not exists stb_1 using stb tags(100)
'''
)
tdLog.printNoPrefix("==========step2:insert data into ntb==========")
tdSql.execute(
'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())')
tdSql.execute(
'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())')
tdLog.printNoPrefix("==========step2:query test of ntb ==========")
# test function today()
# normal table
tdSql.query("select today() from ntb")
tdSql.checkRows(3)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() from db.ntb")
tdSql.checkRows(3)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() +1w from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1w from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1d from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1d from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1h from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1h from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1m from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1m from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1s from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1s from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1a from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1a from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1u from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1u from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1b from ntb")
tdSql.checkRows(3)
tdSql.query("select today() +1b from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1w from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1w from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1d from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1d from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1h from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1h from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1m from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1m from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1s from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1s from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1a from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1a from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1u from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1u from db.ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1b from ntb")
tdSql.checkRows(3)
tdSql.query("select today() -1b from db.ntb")
tdSql.checkRows(3)
tdSql.query("select * from ntb where ts=today()")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 3)
tdSql.query("select * from ntb where ts<=today()")
tdSql.checkRows(2)
tdSql.checkData(0, 1, 10)
# for bug
# tdSql.query("select * from ntb where ts<today()")
# tdSql.checkRows(1)
tdSql.query("select * from ntb where ts>=today()")
tdSql.checkRows(2)
tdSql.checkData(0, 1, 3)
# tdSql.query("select * from ntb where ts>today()")
# tdSql.checkRows(1)
tdSql.query("select c4 from ntb where c4=today()")
tdSql.checkRows(1)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() from ntb where ts<now()")
tdSql.checkRows(3)
tdSql.checkData(0, 0, str(today_date))
tdSql.error("select today()+1.5 from ntb")
tdSql.error("select today()-1.5 from ntb")
tdSql.error("select today()*1.5 from ntb")
tdSql.error("select today()/1.5 from ntb")
tdSql.error("select today()+1.5 from db.ntb")
tdSql.error("select today()-1.5 from db.ntb")
tdSql.error("select today()*1.5 from db.ntb")
tdSql.error("select today()/1.5 from db.ntb")
tdSql.query("select today()+null from ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()+null from db.ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()-null from ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()-null from db.ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()*null from ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()*null from db.ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()/null from ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()/null from db.ntb")
tdSql.checkData(0,0,None)
tdSql.query("select today()/0 from db.ntb")
tdSql.checkRows(3)
tdSql.checkData(0,0,None)
tdSql.checkData(1,0,None)
tdSql.checkData(2,0,None)
# stable
tdSql.query("select today() from stb")
tdSql.checkRows(3)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() +1w from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1w from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1d from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1d from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1h from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1h from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1m from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1m from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1s from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1s from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1a from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1a from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1u from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1u from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() +1b from stb")
tdSql.checkRows(3)
tdSql.query("select today() +1b from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1w from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1w from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1d from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1d from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1h from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1h from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1m from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1m from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1s from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1s from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1a from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1a from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1u from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1u from db.stb")
tdSql.checkRows(3)
tdSql.query("select today() -1b from stb")
tdSql.checkRows(3)
tdSql.query("select today() -1b from db.stb")
tdSql.checkRows(3)
tdSql.query("select ts from stb where ts=today()")
tdSql.checkRows(1)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select ts from stb where ts<=today()")
tdSql.checkRows(2)
tdSql.error("select today()+1.5 from stb")
tdSql.error("select today()-1.5 from stb")
tdSql.error("select today()*1.5 from stb")
tdSql.error("select today()/1.5 from stb")
tdSql.query("select today()+null from stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()+null from db.stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()-null from stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()-null from db.stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()*null from stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()*null from db.stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()/null from stb")
tdSql.checkData(0,0,None)
tdSql.query("select today()/null from db.stb")
tdSql.checkData(0,0,None)
#
# tdSql.query("select * from ntb where ts<today()")
# tdSql.checkRows(1)
# tdSql.query("select * from stb where ts>=today()")
# tdSql.checkRows(2)
# tdSql.query("select * from ntb where ts>today()")
# tdSql.checkRows(1)
tdSql.query("select c4 from stb where c4=today()")
tdSql.checkRows(1)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() from stb where ts<now()")
tdSql.checkRows(3)
tdSql.query("select today()/0 from db.stb")
tdSql.checkRows(3)
tdSql.checkData(0,0,None)
tdSql.checkData(1,0,None)
tdSql.checkData(2,0,None)
# table
tdSql.query("select today() from stb_1")
tdSql.checkRows(3)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() from db.stb_1")
tdSql.checkRows(3)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select today() +1w from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1w from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1d from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1d from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1h from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1h from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1m from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1m from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1s from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1s from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1a from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1a from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1u from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1u from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1b from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() +1b from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1w from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1w from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1d from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1d from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1h from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1h from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1m from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1m from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1s from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1s from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1a from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1a from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1u from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1u from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1b from stb_1")
tdSql.checkRows(3)
tdSql.query("select today() -1b from db.stb_1")
tdSql.checkRows(3)
tdSql.query("select ts from stb_1 where ts=today()")
tdSql.checkRows(1)
tdSql.checkData(0, 0, str(today_date))
tdSql.query("select ts from stb_1 where ts<=today()")
tdSql.checkRows(2)
# for bug
tdSql.query("select * from ntb where ts<today()")
tdSql.checkRows(1)
tdSql.checkData(0,0,"2020-1-1 00:00:00")
tdSql.query("select * from stb_1 where ts>=today()")
tdSql.checkRows(2)
tdSql.query("select * from ntb where ts>today()")
tdSql.checkRows(1)
tdSql.query("select c4 from stb_1 where c4=today()")
tdSql.checkRows(1)
tdSql.query("select today() from stb_1 where ts<now()")
tdSql.checkRows(3)
tdSql.error("select today()+1.5 from stb_1")
tdSql.error("select today()-1.5 from stb_1")
tdSql.error("select today()*1.5 from stb_1")
tdSql.error("select today()/1.5 from stb_1")
tdSql.query("select today()+null from stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()+null from db.stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()-null from stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()-null from db.stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()*null from stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()*null from db.stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()/null from stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()/null from db.stb_1")
tdSql.checkData(0,0,None)
tdSql.query("select today()/0 from db.stb_1")
tdSql.checkRows(3)
tdSql.checkData(0,0,None)
tdSql.checkData(1,0,None)
tdSql.checkData(2,0,None)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

206
tests/system-test/2-query/ttl_comment.py Normal file
View File

@ -0,0 +1,206 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, db_test.stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
class TDTestCase:
def caseDescription(self):
'''
ttl/comment test
'''
return
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.prepare()
tdSql.error("create table ttl_table1(ts timestamp, i int) ttl 1.1")
tdSql.error("create table ttl_table2(ts timestamp, i int) ttl 1e1")
tdSql.error("create table ttl_table3(ts timestamp, i int) ttl -1")
print("============== STEP 1 ===== test normal table")
tdSql.execute("create table normal_table1(ts timestamp, i int)")
tdSql.execute("create table normal_table2(ts timestamp, i int) comment '' ttl 3")
tdSql.execute("create table normal_table3(ts timestamp, i int) ttl 2100000000020 comment 'hello'")
tdSql.query("show tables like 'normal_table1'")
tdSql.checkData(0, 0, 'normal_table1')
tdSql.checkData(0, 7, 0)
tdSql.checkData(0, 8, None)
tdSql.query("show tables like 'normal_table2'")
tdSql.checkData(0, 0, 'normal_table2')
tdSql.checkData(0, 7, 3)
tdSql.checkData(0, 8, '')
tdSql.query("show tables like 'normal_table3'")
tdSql.checkData(0, 0, 'normal_table3')
tdSql.checkData(0, 7, 2147483647)
tdSql.checkData(0, 8, 'hello')
tdSql.execute("alter table normal_table1 comment 'nihao'")
tdSql.query("show tables like 'normal_table1'")
tdSql.checkData(0, 0, 'normal_table1')
tdSql.checkData(0, 8, 'nihao')
tdSql.execute("alter table normal_table1 comment ''")
tdSql.query("show tables like 'normal_table1'")
tdSql.checkData(0, 0, 'normal_table1')
tdSql.checkData(0, 8, '')
tdSql.execute("alter table normal_table2 comment 'fly'")
tdSql.query("show tables like 'normal_table2'")
tdSql.checkData(0, 0, 'normal_table2')
tdSql.checkData(0, 8, 'fly')
tdSql.execute("alter table normal_table3 comment 'fly'")
tdSql.query("show tables like 'normal_table3'")
tdSql.checkData(0, 0, 'normal_table3')
tdSql.checkData(0, 8, 'fly')
tdSql.execute("alter table normal_table1 ttl 1")
tdSql.query("show tables like 'normal_table1'")
tdSql.checkData(0, 0, 'normal_table1')
tdSql.checkData(0, 7, 1)
tdSql.execute("alter table normal_table3 ttl 0")
tdSql.query("show tables like 'normal_table3'")
tdSql.checkData(0, 0, 'normal_table3')
tdSql.checkData(0, 7, 0)
print("============== STEP 2 ===== test super table")
tdSql.execute("create table super_table1(ts timestamp, i int) tags(t int)")
tdSql.execute("create table super_table2(ts timestamp, i int) tags(t int) comment ''")
tdSql.execute("create table super_table3(ts timestamp, i int) tags(t int) comment 'super'")
tdSql.query("show stables like 'super_table1'")
tdSql.checkData(0, 0, 'super_table1')
tdSql.checkData(0, 6, None)
tdSql.query("show stables like 'super_table2'")
tdSql.checkData(0, 0, 'super_table2')
tdSql.checkData(0, 6, '')
tdSql.query("show stables like 'super_table3'")
tdSql.checkData(0, 0, 'super_table3')
tdSql.checkData(0, 6, 'super')
tdSql.execute("alter table super_table1 comment 'nihao'")
tdSql.query("show stables like 'super_table1'")
tdSql.checkData(0, 0, 'super_table1')
tdSql.checkData(0, 6, 'nihao')
tdSql.execute("alter table super_table1 comment ''")
tdSql.query("show stables like 'super_table1'")
tdSql.checkData(0, 0, 'super_table1')
tdSql.checkData(0, 6, '')
tdSql.execute("alter table super_table2 comment 'fly'")
tdSql.query("show stables like 'super_table2'")
tdSql.checkData(0, 0, 'super_table2')
tdSql.checkData(0, 6, 'fly')
tdSql.execute("alter table super_table3 comment 'tdengine'")
tdSql.query("show stables like 'super_table3'")
tdSql.checkData(0, 0, 'super_table3')
tdSql.checkData(0, 6, 'tdengine')
print("============== STEP 3 ===== test child table")
tdSql.execute("create table child_table1 using super_table1 tags(1) ttl 10")
tdSql.execute("create table child_table2 using super_table1 tags(1) comment ''")
tdSql.execute("create table child_table3 using super_table1 tags(1) comment 'child'")
tdSql.execute("insert into child_table4 using super_table1 tags(1) values(now, 1)")
tdSql.query("show tables like 'child_table1'")
tdSql.checkData(0, 0, 'child_table1')
tdSql.checkData(0, 7, 10)
tdSql.checkData(0, 8, None)
tdSql.query("show tables like 'child_table2'")
tdSql.checkData(0, 0, 'child_table2')
tdSql.checkData(0, 7, 0)
tdSql.checkData(0, 8, '')
tdSql.query("show tables like 'child_table3'")
tdSql.checkData(0, 0, 'child_table3')
tdSql.checkData(0, 8, 'child')
tdSql.query("show tables like 'child_table4'")
tdSql.checkData(0, 0, 'child_table4')
tdSql.checkData(0, 7, 0)
tdSql.checkData(0, 8, None)
tdSql.execute("alter table child_table1 comment 'nihao'")
tdSql.query("show tables like 'child_table1'")
tdSql.checkData(0, 0, 'child_table1')
tdSql.checkData(0, 8, 'nihao')
tdSql.execute("alter table child_table1 comment ''")
tdSql.query("show tables like 'child_table1'")
tdSql.checkData(0, 0, 'child_table1')
tdSql.checkData(0, 8, '')
tdSql.execute("alter table child_table2 comment 'fly'")
tdSql.query("show tables like 'child_table2'")
tdSql.checkData(0, 0, 'child_table2')
tdSql.checkData(0, 8, 'fly')
tdSql.execute("alter table child_table3 comment 'tdengine'")
tdSql.query("show tables like 'child_table3'")
tdSql.checkData(0, 0, 'child_table3')
tdSql.checkData(0, 8, 'tdengine')
tdSql.execute("alter table child_table4 comment 'tdengine'")
tdSql.query("show tables like 'child_table4'")
tdSql.checkData(0, 0, 'child_table4')
tdSql.checkData(0, 8, 'tdengine')
tdSql.execute("alter table child_table4 ttl 9")
tdSql.query("show tables like 'child_table4'")
tdSql.checkData(0, 0, 'child_table4')
tdSql.checkData(0, 7, 9)
tdSql.execute("alter table child_table3 ttl 9")
tdSql.query("show tables like 'child_table3'")
tdSql.checkData(0, 0, 'child_table3')
tdSql.checkData(0, 7, 9)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

116
tests/system-test/7-tmq/subscribeDb4.py Normal file
View File

@ -0,0 +1,116 @@
import sys
import time
import socket
import os
import threading
import taos
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
paraDict = {'dbName': 'db12',
'dropFlag':1,
'vgroups': 4,
'precision': 'ms',
'stbName': 'stb0',
'ctbNum': 10,
'rowsPerTbl': 10000,
'batchNum': 10,
'startTs': 0, # 1640966400000 ----> 2022-01-01 00:00:00.000
'event':'',
'columnDict': {'int':2},
'tagDict': {'int':1}
}
cdbName = 'cdb'
# some parameter to consumer processor
consumerId = 0
expectrowcnt = 0
topicList = ''
ifcheckdata = 0
ifManualCommit = 1
groupId = 'group.id:cgrp1'
autoCommit = 'enable.auto.commit:false'
autoCommitInterval = 'auto.commit.interval.ms:1000'
autoOffset = 'auto.offset.reset:earliest'
pollDelay = 20
showMsg = 1
showRow = 1
hostname = socket.gethostname()
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
logSql = False
tdSql.init(conn.cursor(), logSql)
def tmqCase12(self):
tdLog.printNoPrefix("======== test case 12: ")
tdLog.info("step 1: create database, stb, ctb and insert data")
tmqCom.initConsumerTable(self.cdbName)
tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"], self.paraDict['precision'])
self.paraDict["stbName"] = 'stb1'
tdCom.create_stable(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["columnDict"],self.paraDict["tagDict"])
tdCom.create_ctables(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["tagDict"])
tdCom.insert_data(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"])
self.paraDict["stbName"] = 'stb2'
tdCom.create_stable(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["columnDict"],self.paraDict["tagDict"])
tdCom.create_ctables(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["tagDict"])
tdCom.insert_data(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"])
tdLog.info("create topics from db")
topicName1 = 'topic_%s'%(self.paraDict['dbName'])
tdSql.execute("create topic %s as database %s" %(topicName1, self.paraDict['dbName']))
topicList = topicName1
keyList = '%s,%s,%s,%s'%(self.groupId,self.autoCommit,self.autoCommitInterval,self.autoOffset)
self.expectrowcnt = self.paraDict["rowsPerTbl"] * self.paraDict["ctbNum"] * 2
tmqCom.insertConsumerInfo(self.consumerId, self.expectrowcnt,topicList,keyList,self.ifcheckdata,self.ifManualCommit)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName)
tdLog.info("After waiting for a period of time, drop one stable")
time.sleep(10)
tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName']))
tdLog.info("wait result from consumer, then check it")
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
totalConsumeRows = 0
for i in range(expectRows):
totalConsumeRows += resultList[i]
if not (totalConsumeRows >= self.expectrowcnt/2 and totalConsumeRows <= self.expectrowcnt):
tdLog.info("act consume rows: %d, expect consume rows: between %d and %d"%(totalConsumeRows, self.expectrowcnt/2, self.expectrowcnt))
tdLog.exit("tmq consume rows error!")
time.sleep(15)
tdSql.query("drop topic %s"%topicName1)
tdLog.printNoPrefix("======== test case 12 end ...... ")
def run(self):
tdSql.prepare()
self.tmqCase12()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

22
tests/system-test/7-tmq/subscribeStb4.py
View File

@ -196,16 +196,16 @@ class TDTestCase:
auotCtbPrefix = 'autoCtb'
# create and start thread
parameterDict = {'cfg': '', \
'actionType': 0, \
'dbName': 'db1', \
'dropFlag': 1, \
'vgroups': 4, \
'replica': 1, \
'stbName': 'stb1', \
'ctbNum': 10, \
'rowsPerTbl': 10000, \
'batchNum': 100, \
parameterDict = {'cfg': '',
'actionType': 0,
'dbName': 'db1',
'dropFlag': 1,
'vgroups': 4,
'replica': 1,
'stbName': 'stb1',
'ctbNum': 10,
'rowsPerTbl': 10000,
'batchNum': 100,
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
parameterDict['cfg'] = cfgPath
@ -349,7 +349,5 @@ class TDTestCase:
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

118
tests/system-test/7-tmq/tmqCommon.py Normal file
View File

@ -0,0 +1,118 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from collections import defaultdict
import random
import string
import threading
import requests
import time
# import socketfrom
import taos
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
# class actionType(Enum):
# CREATE_DATABASE = 0
# CREATE_STABLE = 1
# CREATE_CTABLE = 2
# INSERT_DATA = 3
class TMQCom:
def init(self, conn, logSql):
tdSql.init(conn.cursor())
# tdSql.init(conn.cursor(), logSql) # output sql.txt file
def initConsumerTable(self,cdbName='cdb'):
tdLog.info("create consume database, and consume info table, and consume result table")
tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
def initConsumerInfoTable(self,cdbName='cdb'):
tdLog.info("drop consumeinfo table")
tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'):
sql = "insert into %s.consumeinfo values "%cdbName
sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
tdLog.info("consume info sql: %s"%sql)
tdSql.query(sql)
def selectConsumeResult(self,expectRows,cdbName='cdb'):
resultList=[]
while 1:
tdSql.query("select * from %s.consumeresult"%cdbName)
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if tdSql.getRows() == expectRows:
break
else:
time.sleep(5)
for i in range(expectRows):
tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3)))
resultList.append(tdSql.getData(i , 3))
return resultList
def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
if valgrind == 1:
logFile = cfgPath + '/../log/valgrind-tmq.log'
shellCmd = 'nohup valgrind --log-file=' + logFile
shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
if (platform.system().lower() == 'windows'):
shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
shellCmd += "> nul 2>&1 &"
else:
shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
shellCmd += "> /dev/null 2>&1 &"
tdLog.info(shellCmd)
os.system(shellCmd)
def syncCreateDbStbCtbInsertData(self, tsql, paraDict):
tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"], paraDict['precision'])
tdCom.create_stable(tsql, paraDict["dbName"],paraDict["stbName"], paraDict["columnDict"], paraDict["tagDict"])
tdCom.create_ctables(tsql, paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["tagDict"])
if "event" in paraDict and type(paraDict['event']) == type(threading.Event()):
paraDict["event"].set()
tdCom.insert_data(tsql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
return
def threadFunction(self, **paraDict):
# create new connector for new tdSql instance in my thread
newTdSql = tdCom.newTdSql()
self.syncCreateDbStbCtbInsertData(self, newTdSql, paraDict)
return
def asyncCreateDbStbCtbInsertData(self, paraDict):
pThread = threading.Thread(target=self.threadFunction, kwargs=paraDict)
pThread.start()
return pThread
def close(self):
self.cursor.close()
tmqCom = TMQCom()

1
tests/system-test/fulltest.sh
View File

@ -98,6 +98,7 @@ python3 ./test.py -f 2-query/stateduration.py
python3 ./test.py -f 2-query/function_stateduration.py
python3 ./test.py -f 2-query/statecount.py
python3 ./test.py -f 2-query/tail.py
python3 ./test.py -f 2-query/ttl_comment.py
python3 ./test.py -f 2-query/distribute_agg_count.py
python3 ./test.py -f 2-query/distribute_agg_max.py
python3 ./test.py -f 2-query/distribute_agg_min.py