xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:[TS-5441] conflicts from 3.0

This commit is contained in:
wangmm0220 2024-10-10 18:32:25 +08:00
parent 42b7520d34 9fef5393e6
commit a9f393aa56
108 changed files with 4377 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmake/cmake.options
View File

@ -144,6 +144,12 @@ option(
OFF
)
option(
BUILD_WITH_ANALYSIS
"If build with analysis"
ON
)
ENDIF ()
IF(NOT TD_ENTERPRISE)
@ -151,6 +157,13 @@ MESSAGE("switch s3 off with community version")
set(BUILD_S3 OFF)
set(BUILD_WITH_S3 OFF)
set(BUILD_WITH_COS OFF)
set(BUILD_WITH_ANALYSIS OFF)
ENDIF ()
IF(${BUILD_WITH_ANALYSIS})
message("build with analysis")
set(BUILD_S3 ON)
set(BUILD_WITH_S3 ON)
ENDIF()
IF(${BUILD_S3})

2
docs/zh/04-get-started/05-cloud.md
View File

@ -15,7 +15,7 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担
要在 TDengine Cloud 注册新用户，请遵循以下简易步骤完成注册流程：
1. 打开浏览器，访问 TDengine Cloud 的首页：https://cloud.taosdata.com，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。
1. 打开浏览器，访问 [TDengine Cloud](https://cloud.taosdata.com)，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。
2. 检查企业邮箱，找到主题为“你的 TDengine Cloud 注册账户验证码”的邮件。从邮件内容中复制 6 位验证码，并将其粘贴到注册页面上的“验证码”输入框中。接着，点击“注册 TDengine Cloud”按钮，进入客户信息补全页面。

4
docs/zh/08-operation/03-deployment.md
View File

@ -206,11 +206,11 @@ http {
### 部署 taosX
如果想使用 TDengine 的数据接入能力，需要部署 taosX 服务，关于它的详细说明和部署请参考[taosX 参考手册](../../reference/components/taosx)。
如果想使用 TDengine 的数据接入能力，需要部署 taosX 服务，关于它的详细说明和部署请参考企业版参考手册。
### 部署 taosX-Agent
有些数据源如 Pi, OPC 等，因为网络条件和数据源访问的限制，taosX 无法直接访问数据源，这种情况下需要部署一个代理服务 taosX-Agent，关于它的详细说明和部署请参考[taosX-Agent 参考手册](../../reference/components/taosx-agent)。
有些数据源如 Pi, OPC 等，因为网络条件和数据源访问的限制，taosX 无法直接访问数据源，这种情况下需要部署一个代理服务 taosX-Agent，关于它的详细说明和部署请参考企业版参考手册。
### 部署 taos-Explorer

2
docs/zh/08-operation/12-multi.md
View File

@ -70,7 +70,7 @@ dataDir /mnt/data6 2 0
|参数名称 | 参数含义 |
|:-------------|:-----------------------------------------------|
|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。例如：http://cos.ap-beijing.myqcloud.com |
|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 |
|s3AccessKey |冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E |
|s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 |
|s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 |

2
docs/zh/08-operation/18-dual.md
View File

@ -83,7 +83,7 @@ taosx replica start
```shell
taosx replica start -f td1:6030 -t td2:6030
```
该示例命令会自动创建除 information_schema、performance_schema、log、audit 库之外的同步任务。可以使用 http://td2:6041 指定该 endpoint 使用 websocket 接口（默认是原生接口）。也可以指定数据库同步：taosx replica start -f td1:6030 -t td2:6030 db1 仅创建指定的数据库同步任务。
该示例命令会自动创建除 information_schema、performance_schema、log、audit 库之外的同步任务。可以使用 `http://td2:6041` 指定该 endpoint 使用 websocket 接口（默认是原生接口）。也可以指定数据库同步：taosx replica start -f td1:6030 -t td2:6030 db1 仅创建指定的数据库同步任务。
2. 方法二

2
docs/zh/14-reference/03-taos-sql/14-stream.md
View File

@ -99,7 +99,7 @@ PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子
## 流式计算读取历史数据
正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。例如：
正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。流计算处理历史数据的最大窗口数是2000万，超过限制会报错。例如：
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s)

2
include/common/systable.h
View File

@ -29,6 +29,8 @@ extern "C" {
#define TSDB_INS_TABLE_QNODES "ins_qnodes"
#define TSDB_INS_TABLE_BNODES "ins_bnodes" // no longer used
#define TSDB_INS_TABLE_SNODES "ins_snodes"
#define TSDB_INS_TABLE_ANODES "ins_anodes"
#define TSDB_INS_TABLE_ANODES_FULL "ins_anodes_full"
#define TSDB_INS_TABLE_ARBGROUPS "ins_arbgroups"
#define TSDB_INS_TABLE_CLUSTER "ins_cluster"
#define TSDB_INS_TABLE_DATABASES "ins_databases"

96
include/common/tanal.h Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_ANAL_H_
#define _TD_UTIL_ANAL_H_
#include "os.h"
#include "tdef.h"
#include "thash.h"
#include "tjson.h"
#ifdef __cplusplus
extern "C" {
#endif
#define ANAL_FORECAST_DEFAULT_PERIOD 10
#define ANAL_FORECAST_DEFAULT_ROWS 10
#define ANAL_FORECAST_DEFAULT_CONF 95
#define ANAL_FORECAST_DEFAULT_ALPHA 0.05
#define ANAL_FORECAST_DEFAULT_PARAM "diff"
typedef struct {
EAnalAlgoType type;
int32_t anode;
int32_t urlLen;
char *url;
} SAnalUrl;
typedef enum {
ANAL_BUF_TYPE_JSON = 0,
ANAL_BUF_TYPE_JSON_COL = 1,
ANAL_BUF_TYPE_OTHERS,
} EAnalBufType;
typedef enum {
ANAL_HTTP_TYPE_GET = 0,
ANAL_HTTP_TYPE_POST,
} EAnalHttpType;
typedef struct {
TdFilePtr filePtr;
char fileName[TSDB_FILENAME_LEN + 10];
int64_t numOfRows;
} SAnalColBuf;
typedef struct {
EAnalBufType bufType;
TdFilePtr filePtr;
char fileName[TSDB_FILENAME_LEN];
int32_t numOfCols;
SAnalColBuf *pCols;
} SAnalBuf;
int32_t taosAnalInit();
void taosAnalCleanup();
SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf);
int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen);
bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen);
bool taosAnalGetOptInt(const char *option, const char *optName, int32_t *optValue);
int64_t taosAnalGetVersion();
void taosAnalUpdate(int64_t newVer, SHashObj *pHash);
int32_t tsosAnalBufOpen(SAnalBuf *pBuf, int32_t numOfCols);
int32_t taosAnalBufWriteOptStr(SAnalBuf *pBuf, const char *optName, const char *optVal);
int32_t taosAnalBufWriteOptInt(SAnalBuf *pBuf, const char *optName, int64_t optVal);
int32_t taosAnalBufWriteOptFloat(SAnalBuf *pBuf, const char *optName, float optVal);
int32_t taosAnalBufWriteColMeta(SAnalBuf *pBuf, int32_t colIndex, int32_t colType, const char *colName);
int32_t taosAnalBufWriteDataBegin(SAnalBuf *pBuf);
int32_t taosAnalBufWriteColBegin(SAnalBuf *pBuf, int32_t colIndex);
int32_t taosAnalBufWriteColData(SAnalBuf *pBuf, int32_t colIndex, int32_t colType, void *colValue);
int32_t taosAnalBufWriteColEnd(SAnalBuf *pBuf, int32_t colIndex);
int32_t taosAnalBufWriteDataEnd(SAnalBuf *pBuf);
int32_t taosAnalBufClose(SAnalBuf *pBuf);
void taosAnalBufDestroy(SAnalBuf *pBuf);
const char *taosAnalAlgoStr(EAnalAlgoType algoType);
EAnalAlgoType taosAnalAlgoInt(const char *algoName);
const char *taosAnalAlgoUrlStr(EAnalAlgoType algoType);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_ANAL_H_*/

2
include/common/tglobal.h
View File

@ -214,7 +214,7 @@ extern int64_t tsMinDiskFreeSize;
// udf
extern bool tsStartUdfd;
extern char tsUdfdResFuncs[];
extern char tsUdfdLdLibPath[];
extern char tsUdfdLdLibPath[512];
// schemaless
extern char tsSmlChildTableName[];

56
include/common/tmsg.h
View File

@ -159,6 +159,8 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_ENCRYPTIONS,
TSDB_MGMT_TABLE_USER_FULL,
TSDB_MGMT_TABLE_ANODE,
TSDB_MGMT_TABLE_ANODE_FULL,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -260,6 +262,7 @@ typedef enum ENodeType {
QUERY_NODE_COUNT_WINDOW,
QUERY_NODE_COLUMN_OPTIONS,
QUERY_NODE_TSMA_OPTIONS,
QUERY_NODE_ANOMALY_WINDOW,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
@ -345,6 +348,9 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_VIEW_STMT,
QUERY_NODE_DROP_VIEW_STMT,
QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE,
QUERY_NODE_CREATE_ANODE_STMT,
QUERY_NODE_DROP_ANODE_STMT,
QUERY_NODE_UPDATE_ANODE_STMT,
// show statement nodes
// see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
@ -386,6 +392,8 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_SHOW_ANODES_STMT,
QUERY_NODE_SHOW_ANODES_FULL_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
@ -408,6 +416,7 @@ typedef enum ENodeType {
QUERY_NODE_LOGIC_PLAN,
QUERY_NODE_LOGIC_PLAN_GROUP_CACHE,
QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL,
QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC,
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
@ -458,6 +467,9 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY,
QUERY_NODE_PHYSICAL_PLAN_STREAM_ANOMALY,
QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
} ENodeType;
typedef struct {
@ -1092,6 +1104,22 @@ typedef struct {
int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
typedef struct {
int32_t dnodeId;
int64_t analVer;
} SRetrieveAnalAlgoReq;
typedef struct {
int64_t ver;
SHashObj* hash; // algoname:algotype -> SAnalUrl
} SRetrieveAnalAlgoRsp;
int32_t tSerializeRetrieveAnalAlgoReq(void* buf, int32_t bufLen, SRetrieveAnalAlgoReq* pReq);
int32_t tDeserializeRetrieveAnalAlgoReq(void* buf, int32_t bufLen, SRetrieveAnalAlgoReq* pReq);
int32_t tSerializeRetrieveAnalAlgoRsp(void* buf, int32_t bufLen, SRetrieveAnalAlgoRsp* pRsp);
int32_t tDeserializeRetrieveAnalAlgoRsp(void* buf, int32_t bufLen, SRetrieveAnalAlgoRsp* pRsp);
void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp* pRsp);
typedef struct {
int8_t alterType;
int8_t superUser;
@ -1766,6 +1794,7 @@ typedef struct {
SArray* pVloads; // array of SVnodeLoad
int32_t statusSeq;
int64_t ipWhiteVer;
int64_t analVer;
} SStatusReq;
int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
@ -1831,6 +1860,7 @@ typedef struct {
SArray* pDnodeEps; // Array of SDnodeEp
int32_t statusSeq;
int64_t ipWhiteVer;
int64_t analVer;
} SStatusRsp;
int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
@ -2377,6 +2407,30 @@ typedef struct {
int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq);
int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq);
typedef struct {
int32_t urlLen;
int32_t sqlLen;
char* url;
char* sql;
} SMCreateAnodeReq;
int32_t tSerializeSMCreateAnodeReq(void* buf, int32_t bufLen, SMCreateAnodeReq* pReq);
int32_t tDeserializeSMCreateAnodeReq(void* buf, int32_t bufLen, SMCreateAnodeReq* pReq);
void tFreeSMCreateAnodeReq(SMCreateAnodeReq* pReq);
typedef struct {
int32_t anodeId;
int32_t sqlLen;
char* sql;
} SMDropAnodeReq, SMUpdateAnodeReq;
int32_t tSerializeSMDropAnodeReq(void* buf, int32_t bufLen, SMDropAnodeReq* pReq);
int32_t tDeserializeSMDropAnodeReq(void* buf, int32_t bufLen, SMDropAnodeReq* pReq);
void tFreeSMDropAnodeReq(SMDropAnodeReq* pReq);
int32_t tSerializeSMUpdateAnodeReq(void* buf, int32_t bufLen, SMUpdateAnodeReq* pReq);
int32_t tDeserializeSMUpdateAnodeReq(void* buf, int32_t bufLen, SMUpdateAnodeReq* pReq);
void tFreeSMUpdateAnodeReq(SMUpdateAnodeReq* pReq);
typedef struct {
int32_t vgId;
int32_t hbSeq;
@ -4059,7 +4113,7 @@ typedef struct {
SArray* blockData;
SArray* blockTbName;
SArray* blockSchema;
SArray* blockSuid;
// SArray* blockSuid;
union{
struct{

5
include/common/tmsgdef.h
View File

@ -125,6 +125,11 @@
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
// mnode msg overload
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ANODE, "create-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_ANODE, "update-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_ANODE, "drop-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_ANAL_ALGO, "retrieve-anal-algo", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_DND_MSG)
TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8

2
include/libs/command/command.h
View File

@ -29,6 +29,6 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs)
int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp);
int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp);
void qExplainFreeCtx(SExplainCtx *pCtx);
int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes);
int32_t formatDurationOrKeep(char* buffer, int64_t bufSize, int32_t timeInMinutes);
#endif

6
include/libs/function/functionMgt.h
View File

@ -62,6 +62,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_UNIQUE,
FUNCTION_TYPE_STATE_COUNT,
FUNCTION_TYPE_STATE_DURATION,
FUNCTION_TYPE_FORECAST,
// math function
FUNCTION_TYPE_ABS = 1000,
@ -149,6 +150,9 @@ typedef enum EFunctionType {
FUNCTION_TYPE_TBUID,
FUNCTION_TYPE_VGID,
FUNCTION_TYPE_VGVER,
FUNCTION_TYPE_FORECAST_LOW,
FUNCTION_TYPE_FORECAST_HIGH,
FUNCTION_TYPE_FORECAST_ROWTS,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -263,6 +267,7 @@ bool fmIsForbidSysTableFunc(int32_t funcId);
bool fmIsIntervalInterpoFunc(int32_t funcId);
bool fmIsInterpFunc(int32_t funcId);
bool fmIsLastRowFunc(int32_t funcId);
bool fmIsForecastFunc(int32_t funcId);
bool fmIsNotNullOutputFunc(int32_t funcId);
bool fmIsSelectValueFunc(int32_t funcId);
bool fmIsSystemInfoFunc(int32_t funcId);
@ -272,6 +277,7 @@ bool fmIsMultiRowsFunc(int32_t funcId);
bool fmIsKeepOrderFunc(int32_t funcId);
bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId);
bool fmIsForecastPseudoColumnFunc(int32_t funcId);
bool fmIsGroupKeyFunc(int32_t funcId);
bool fmIsBlockDistFunc(int32_t funcId);
bool fmIsIgnoreNullFunc(int32_t funcId);

15
include/libs/nodes/cmdnodes.h
View File

@ -318,6 +318,21 @@ typedef struct SAlterDnodeStmt {
char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt;
typedef struct {
ENodeType type;
char url[TSDB_ANAL_ANODE_URL_LEN];
} SCreateAnodeStmt;
typedef struct {
ENodeType type;
int32_t anodeId;
} SDropAnodeStmt;
typedef struct {
ENodeType type;
int32_t anodeId;
} SUpdateAnodeStmt;
typedef struct SShowStmt {
ENodeType type;
SNode* pDbName; // SValueNode

22
include/libs/nodes/plannodes.h
View File

@ -204,6 +204,11 @@ typedef struct SInterpFuncLogicNode {
SNode* pTimeSeries; // SColumnNode
} SInterpFuncLogicNode;
typedef struct SForecastFuncLogicNode {
SLogicNode node;
SNodeList* pFuncs;
} SForecastFuncLogicNode;
typedef struct SGroupCacheLogicNode {
SLogicNode node;
bool grpColsMayBeNull;
@ -274,7 +279,8 @@ typedef enum EWindowType {
WINDOW_TYPE_SESSION,
WINDOW_TYPE_STATE,
WINDOW_TYPE_EVENT,
WINDOW_TYPE_COUNT
WINDOW_TYPE_COUNT,
WINDOW_TYPE_ANOMALY
} EWindowType;
typedef enum EWindowAlgorithm {
@ -315,6 +321,8 @@ typedef struct SWindowLogicNode {
int64_t windowCount;
int64_t windowSliding;
SNodeList* pTsmaSubplans;
SNode* pAnomalyExpr;
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -507,6 +515,12 @@ typedef struct SInterpFuncPhysiNode {
SNode* pTimeSeries; // SColumnNode
} SInterpFuncPhysiNode;
typedef struct SForecastFuncPhysiNode {
SPhysiNode node;
SNodeList* pExprs;
SNodeList* pFuncs;
} SForecastFuncPhysiNode;
typedef struct SSortMergeJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
@ -704,6 +718,12 @@ typedef struct SCountWinodwPhysiNode {
typedef SCountWinodwPhysiNode SStreamCountWinodwPhysiNode;
typedef struct SAnomalyWindowPhysiNode {
SWindowPhysiNode window;
SNode* pAnomalyKey;
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
} SAnomalyWindowPhysiNode;
typedef struct SSortPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function

9
include/libs/nodes/querynodes.h
View File

@ -347,6 +347,13 @@ typedef struct SCountWindowNode {
int64_t windowSliding;
} SCountWindowNode;
typedef struct SAnomalyWindowNode {
ENodeType type; // QUERY_NODE_ANOMALY_WINDOW
SNode* pCol; // timestamp primary key
SNode* pExpr;
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
} SAnomalyWindowNode;
typedef enum EFillMode {
FILL_MODE_NONE = 1,
FILL_MODE_VALUE,
@ -442,6 +449,8 @@ typedef struct SSelectStmt {
bool hasTailFunc;
bool hasInterpFunc;
bool hasInterpPseudoColFunc;
bool hasForecastFunc;
bool hasForecastPseudoColFunc;
bool hasLastRowFunc;
bool hasLastFunc;
bool hasTimeLineFunc;

10
include/libs/qcom/query.h
View File

@ -336,7 +336,7 @@ char* jobTaskStatusStr(int32_t status);
SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len);
void parseTagDatatoJson(void* p, char** jsonStr);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
@ -410,25 +410,25 @@ void* getTaskPoolWorkerCb();
#define qFatal(...) \
do { \
if (qDebugFlag & DEBUG_FATAL) { \
taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qError(...) \
do { \
if (qDebugFlag & DEBUG_ERROR) { \
taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qWarn(...) \
do { \
if (qDebugFlag & DEBUG_WARN) { \
taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qInfo(...) \
do { \
if (qDebugFlag & DEBUG_INFO) { \
taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qDebug(...) \

1
include/libs/scalar/scalar.h
View File

@ -139,6 +139,7 @@ int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t forecastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

1
include/os/osString.h
View File

@ -68,6 +68,7 @@ typedef enum { M2C = 0, C2M } ConvType;
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src))
#define TAOS_STRNCAT(_dst, _src, len) ((void)strncat(_dst, _src, len))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);

20
include/util/taoserror.h
View File

@ -476,6 +476,22 @@ int32_t taosGetErrSize();
#define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429)
#define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A)
// anode
#define TSDB_CODE_MND_ANODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0430)
#define TSDB_CODE_MND_ANODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0431)
#define TSDB_CODE_MND_ANODE_TOO_LONG_URL TAOS_DEF_ERROR_CODE(0, 0x0432)
#define TSDB_CODE_MND_ANODE_INVALID_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0433)
#define TSDB_CODE_MND_ANODE_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0434)
#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO TAOS_DEF_ERROR_CODE(0, 0x0435)
#define TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME TAOS_DEF_ERROR_CODE(0, 0x0436)
#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE TAOS_DEF_ERROR_CODE(0, 0x0437)
// analysis
#define TSDB_CODE_ANAL_URL_RSP_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x0440)
#define TSDB_CODE_ANAL_URL_CANT_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0441)
#define TSDB_CODE_ANAL_ALGO_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0442)
#define TSDB_CODE_ANAL_ALGO_NOT_LOAD TAOS_DEF_ERROR_CODE(0, 0x0443)
#define TSDB_CODE_ANAL_BUF_INVALID_TYPE TAOS_DEF_ERROR_CODE(0, 0x0444)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
@ -868,6 +884,10 @@ int32_t taosGetErrSize();
#define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F)
#define TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC TAOS_DEF_ERROR_CODE(0, 0x2680)
#define TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2681)
#define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_TYPE TAOS_DEF_ERROR_CODE(0, 0x2682)
#define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL TAOS_DEF_ERROR_CODE(0, 0x2683)
#define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT TAOS_DEF_ERROR_CODE(0, 0x2684)
#define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x2685)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

13
include/util/tdef.h
View File

@ -293,6 +293,12 @@ typedef enum ELogicConditionType {
#define TSDB_SLOW_QUERY_SQL_LEN 512
#define TSDB_SHOW_SUBQUERY_LEN 1000
#define TSDB_LOG_VAR_LEN 32
#define TSDB_ANAL_ANODE_URL_LEN 128
#define TSDB_ANAL_ALGO_NAME_LEN 64
#define TSDB_ANAL_ALGO_TYPE_LEN 24
#define TSDB_ANAL_ALGO_KEY_LEN (TSDB_ANAL_ALGO_NAME_LEN + 9)
#define TSDB_ANAL_ALGO_URL_LEN (TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN + 1)
#define TSDB_ANAL_ALGO_OPTION_LEN 256
#define TSDB_MAX_EP_NUM 10
@ -603,6 +609,13 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 };
#define MONITOR_TAG_NAME_LEN 100
#define MONITOR_TAG_VALUE_LEN 300
#define MONITOR_METRIC_NAME_LEN 100
typedef enum {
ANAL_ALGO_TYPE_ANOMALY_DETECT = 0,
ANAL_ALGO_TYPE_FORECAST = 1,
ANAL_ALGO_TYPE_END,
} EAnalAlgoType;
#ifdef __cplusplus
}
#endif

2
include/util/tjson.h
View File

@ -68,6 +68,8 @@ int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName);
int32_t tjsonGetObjectName(const SJson* pJson, char** pName);
int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue);
void tjsonGetObjectValueBigInt(const SJson* pJson, int64_t* pVal);
void tjsonGetObjectValueDouble(const SJson* pJson, double* pVal);
int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);

33
packaging/tools/com.taosdata.taos-explorer.plist Normal file
View File

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.tdengine.taos-explorer</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/taos-explorer</string>
</array>
<key>ProcessType</key>
<string>Interactive</string>
<key>Disabled</key>
<false/>
<key>RunAtLoad</key>
<false/>
<key>LaunchOnlyOnce</key>
<false/>
<key>SessionCreate</key>
<true/>
<key>ExitTimeOut</key>
<integer>600</integer>
<key>KeepAlive</key>
<dict>
<key>SuccessfulExit</key>
<false/>
<key>AfterInitialDemand</key>
<true/>
</dict>
<key>Program</key>
<string>/usr/local/bin/taos-explorer</string>
</dict>
</plist>

26
packaging/tools/remove.sh
View File

@ -206,10 +206,17 @@ function clean_log() {
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist || :
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}adapter.plist || :
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}keeper.plist || :
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}-explorer.plist || :
${csudo}launchctl remove com.tdengine.taosd || :
${csudo}launchctl remove com.tdengine.${PREFIX}adapter || :
${csudo}launchctl remove com.tdengine.${PREFIX}keeper || :
${csudo}launchctl remove com.tdengine.${PREFIX}-explorer || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.* > /dev/null 2>&1 || :
}
function remove_data_and_config() {
@ -250,6 +257,12 @@ if [ -e ${install_main_dir}/uninstall_${PREFIX}x.sh ]; then
fi
fi
if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
${csudo}rm -rf /Applications/TDengine.app
fi
remove_bin
clean_header
# Remove lib file
@ -282,10 +295,7 @@ elif echo $osinfo | grep -qwi "centos"; then
# echo "this is centos system"
${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || :
fi
if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
${csudo}rm -rf /Applications/TDengine.app
fi
command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true
echo

4
packaging/tools/tdengine.iss
View File

@ -71,8 +71,8 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt
Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden
Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden
Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64"
Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86"
Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbc_install.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64"
Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbc_install.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86"
[UninstallRun]
RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden

10
source/client/src/clientHb.c
View File

@ -55,7 +55,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
for (int32_t i = 0; i < numOfBatchs; ++i) {
SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i);
if (NULL == rsp) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _return;
}
tscDebug("hb to update user auth, user:%s, version:%d", rsp->user, rsp->version);
@ -217,7 +217,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
for (int32_t i = 0; i < numOfBatchs; ++i) {
SDbHbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
if (NULL == rsp) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _return;
}
if (rsp->useDbRsp) {
@ -291,7 +291,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
for (int32_t i = 0; i < numOfMeta; ++i) {
STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i);
if (NULL == rsp) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _return;
}
if (rsp->numOfColumns < 0) {
@ -313,7 +313,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
for (int32_t i = 0; i < numOfIndex; ++i) {
STableIndexRsp *rsp = taosArrayGet(hbRsp.pIndexRsp, i);
if (NULL == rsp) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _return;
}
TSC_ERR_JRET(catalogUpdateTableIndex(pCatalog, rsp));
@ -354,7 +354,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i);
if (NULL == rsp) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _return;
}
if (rsp->numOfCols < 0) {

2
source/client/src/clientImpl.c
View File

@ -949,7 +949,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog,
for (int32_t i = 0; i < tbNum; ++i) {
STbVerInfo* tbInfo = taosArrayGet(pTbArray, i);
if (NULL == tbInfo) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _return;
}
STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion};

17
source/client/src/clientRawBlockWrite.c
View File

@ -456,15 +456,17 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
cJSON* tvalue = NULL;
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
char* buf = NULL;
int64_t bufSize = 0;
if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) {
buf = taosMemoryCalloc(pTagVal->nData * 2 + 2 + 3, 1);
bufSize = pTagVal->nData * 2 + 2 + 3;
} else {
buf = taosMemoryCalloc(pTagVal->nData + 3, 1);
bufSize = pTagVal->nData + 3;
}
buf = taosMemoryCalloc(bufSize, 1);
RAW_NULL_CHECK(buf);
if (!buf) goto end;
if (dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
if (dataConverToStr(buf, bufSize, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto end;
}
@ -734,13 +736,15 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
goto end;
}
} else {
int64_t bufSize = 0;
if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal * 2 + 2 + 3, 1);
bufSize = vAlterTbReq.nTagVal * 2 + 2 + 3;
} else {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1);
bufSize = vAlterTbReq.nTagVal + 3;
}
buf = taosMemoryCalloc(bufSize, 1);
RAW_NULL_CHECK(buf);
if (dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) !=
if (dataConverToStr(buf, bufSize, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) !=
TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto end;
@ -1944,7 +1948,6 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
int retry = 0;
while(1){
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
uDebug(LOG_ID_TAG " write raw data type:%d block num:%d", LOG_ID_VALUE, type, rspObj.dataRsp.blockNum);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);

8
source/client/src/clientSml.c
View File

@ -393,7 +393,7 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) {
tinfo->tags = taosArrayDup(info->preLineTagKV, NULL);
if (tinfo->tags == NULL) {
smlDestroyTableInfo(&tinfo);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i);
@ -561,7 +561,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) {
if (strlen(oneTable->childTableName) == 0) {
SArray *dst = taosArrayDup(oneTable->tags, NULL);
if (dst == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) {
uError("SML:smlSetCTableName super table name is too long");
@ -957,7 +957,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
for (; i < taosArrayGetSize(cols); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
if (kv == NULL) {
code = TSDB_CODE_SML_INVALID_DATA;
code = terrno;
goto END;
}
if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) {
@ -1053,7 +1053,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns,
for (int32_t i = 0; i < pReq.numOfColumns; ++i) {
SField *pField = taosArrayGet(pColumns, i);
if (pField == NULL) {
code = TSDB_CODE_SML_INVALID_DATA;
code = terrno;
goto end;
}
SFieldWithOptions fieldWithOption = {0};

2
source/client/src/clientStmt.c
View File

@ -983,7 +983,7 @@ int stmtSetDbName(TAOS_STMT* stmt, const char* dbName) {
taosMemoryFreeClear(pStmt->exec.pRequest->pDb);
pStmt->exec.pRequest->pDb = taosStrdup(dbName);
if (pStmt->exec.pRequest->pDb == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
return TSDB_CODE_SUCCESS;
}

2
source/client/src/clientStmt2.c
View File

@ -850,7 +850,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) {
taosMemoryFreeClear(pStmt->exec.pRequest->pDb);
pStmt->exec.pRequest->pDb = taosStrdup(dbName);
if (pStmt->exec.pRequest->pDb == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
return TSDB_CODE_SUCCESS;
}

4
source/client/src/clientTmq.c
View File

@ -826,7 +826,7 @@ static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){
for (int32_t j = 0; j < numOfVgroups; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (pVg == NULL) {
code = TSDB_CODE_INVALID_PARA;
code = terrno;
goto END;
}
@ -978,7 +978,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
req.consumerId = tmq->consumerId;
req.epoch = tmq->epoch;
req.pollFlag = atomic_load_8(&tmq->pollFlag);
tqDebugC("consumer:0x%" PRIx64 " send hb, pollFlag:%d", tmq->consumerId, req.pollFlag);
tqDebugC("consumer:0x%" PRIx64 " send heartbeat, pollFlag:%d", tmq->consumerId, req.pollFlag);
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
if (req.topics == NULL) {
goto END;

4
source/common/CMakeLists.txt
View File

@ -47,6 +47,10 @@ target_link_libraries(
INTERFACE api
)
if(${BUILD_WITH_ANALYSIS})
add_definitions(-DUSE_ANAL)
endif()
if(${BUILD_S3})
if(${BUILD_WITH_S3})

2
source/common/src/rsync.c
View File

@ -94,7 +94,7 @@ static int32_t generateConfigFile(char* confDir) {
#endif
);
uDebug("[rsync] conf:%s", confContent);
if (taosWriteFile(pFile, confContent, strlen(confContent)) != TSDB_CODE_SUCCESS) {
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
(void)taosCloseFile(&pFile);
code = terrno;

17
source/common/src/systable.c
View File

@ -398,6 +398,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema anodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "url", .bytes = TSDB_ANAL_ANODE_URL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "update_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
};
static const SSysDbTableSchema anodesFullSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "type", .bytes = TSDB_ANAL_ALGO_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "algo", .bytes = TSDB_ANAL_ALGO_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema tsmaSchema[] = {
{.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@ -472,6 +487,8 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_ENCRYPTIONS, encryptionsSchema, tListLen(encryptionsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
{TSDB_INS_TABLE_ANODES, anodesSchema, tListLen(anodesSchema), true},
{TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true},
};
static const SSysDbTableSchema connectionsSchema[] = {

2
source/common/src/tglobal.c
View File

@ -362,7 +362,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD
char *strDup = NULL;
if ((strDup = taosStrdup(pItem->str))== NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}

6
source/common/src/tmisce.c
View File

@ -284,7 +284,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
@ -297,7 +297,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
@ -309,7 +309,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit);

264
source/common/src/tmsg.c
View File

@ -40,6 +40,7 @@
#define TD_MSG_RANGE_CODE_
#include "tmsgdef.h"
#include "tanal.h"
#include "tcol.h"
#include "tlog.h"
@ -1453,6 +1454,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
}
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->ipWhiteVer));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->analVer));
TAOS_CHECK_EXIT(tSerializeSMonitorParas(&encoder, &pReq->clusterCfg.monitorParas));
tEndEncode(&encoder);
@ -1576,6 +1578,10 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->ipWhiteVer));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->analVer));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDeserializeSMonitorParas(&decoder, &pReq->clusterCfg.monitorParas));
}
@ -1652,6 +1658,7 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->statusSeq));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->ipWhiteVer));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->analVer));
tEndEncode(&encoder);
_exit:
@ -1703,6 +1710,11 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ipWhiteVer));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->analVer));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
@ -2044,6 +2056,156 @@ _exit:
return code;
}
int32_t tSerializeRetrieveAnalAlgoReq(void *buf, int32_t bufLen, SRetrieveAnalAlgoReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->dnodeId));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->analVer));
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeRetrieveAnalAlgoReq(void *buf, int32_t bufLen, SRetrieveAnalAlgoReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->dnodeId));
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->analVer));
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAlgoRsp *pRsp) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
int32_t numOfAlgos = 0;
void *pIter = taosHashIterate(pRsp->hash, NULL);
while (pIter != NULL) {
SAnalUrl *pUrl = pIter;
size_t nameLen = 0;
const char *name = taosHashGetKey(pIter, &nameLen);
if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_KEY_LEN && pUrl->urlLen > 0) {
numOfAlgos++;
}
pIter = taosHashIterate(pRsp->hash, pIter);
}
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->ver));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, numOfAlgos));
pIter = taosHashIterate(pRsp->hash, NULL);
while (pIter != NULL) {
SAnalUrl *pUrl = pIter;
size_t nameLen = 0;
const char *name = taosHashGetKey(pIter, &nameLen);
if (nameLen > 0 && pUrl->urlLen > 0) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, nameLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)name, nameLen));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pUrl->anode));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pUrl->type));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pUrl->urlLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)pUrl->url, pUrl->urlLen));
}
pIter = taosHashIterate(pRsp->hash, pIter);
}
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAlgoRsp *pRsp) {
if (pRsp->hash == NULL) {
pRsp->hash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (pRsp->hash == NULL) {
terrno = TSDB_CODE_OUT_OF_BUFFER;
return terrno;
}
}
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
int32_t numOfAlgos = 0;
int32_t nameLen;
int32_t type;
char name[TSDB_ANAL_ALGO_KEY_LEN];
SAnalUrl url = {0};
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ver));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &numOfAlgos));
for (int32_t f = 0; f < numOfAlgos; ++f) {
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &nameLen));
if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_NAME_LEN) {
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, name));
}
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &url.anode));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &type));
url.type = (EAnalAlgoType)type;
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &url.urlLen));
if (url.urlLen > 0) {
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&url.url, NULL) < 0);
}
TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalUrl)));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp *pRsp) {
void *pIter = taosHashIterate(pRsp->hash, NULL);
while (pIter != NULL) {
SAnalUrl *pUrl = (SAnalUrl *)pIter;
taosMemoryFree(pUrl->url);
pIter = taosHashIterate(pRsp->hash, pIter);
}
taosHashCleanup(pRsp->hash);
pRsp->hash = NULL;
}
void tFreeSCreateUserReq(SCreateUserReq *pReq) {
FREESQL();
taosMemoryFreeClear(pReq->pIpRanges);
@ -2961,6 +3123,108 @@ _exit:
return code;
}
int32_t tSerializeSMCreateAnodeReq(void *buf, int32_t bufLen, SMCreateAnodeReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->urlLen));
if (pReq->urlLen > 0) {
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (const uint8_t *)pReq->url, pReq->urlLen));
}
ENCODESQL();
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMCreateAnodeReq(void *buf, int32_t bufLen, SMCreateAnodeReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->urlLen));
if (pReq->urlLen > 0) {
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pReq->url, NULL));
}
DECODESQL();
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeSMCreateAnodeReq(SMCreateAnodeReq *pReq) {
taosMemoryFreeClear(pReq->url);
FREESQL();
}
int32_t tSerializeSMDropAnodeReq(void *buf, int32_t bufLen, SMDropAnodeReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->anodeId));
ENCODESQL();
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMDropAnodeReq(void *buf, int32_t bufLen, SMDropAnodeReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->anodeId));
DECODESQL();
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeSMDropAnodeReq(SMDropAnodeReq *pReq) { FREESQL(); }
int32_t tSerializeSMUpdateAnodeReq(void *buf, int32_t bufLen, SMUpdateAnodeReq *pReq) {
return tSerializeSMDropAnodeReq(buf, bufLen, pReq);
}
int32_t tDeserializeSMUpdateAnodeReq(void *buf, int32_t bufLen, SMUpdateAnodeReq *pReq) {
return tDeserializeSMDropAnodeReq(buf, bufLen, pReq);
}
void tFreeSMUpdateAnodeReq(SMUpdateAnodeReq *pReq) { tFreeSMDropAnodeReq(pReq); }
int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;

45
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -18,6 +18,7 @@
#include "dmInt.h"
#include "monitor.h"
#include "systable.h"
#include "tanal.h"
#include "tchecksum.h"
extern SConfig *tsCfg;
@ -39,6 +40,7 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
(void)taosThreadRwlockUnlock(&pMgmt->pData->lock);
}
}
static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
int32_t code = 0;
dDebug("ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver);
@ -84,6 +86,47 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
dError("failed to send retrieve ip white list request since:%s", tstrerror(code));
}
}
static void dmMayShouldUpdateAnalFunc(SDnodeMgmt *pMgmt, int64_t newVer) {
int32_t code = 0;
int64_t oldVer = taosAnalGetVersion();
if (oldVer == newVer) return;
dDebug("analysis on dnode ver:%" PRId64 ", status ver:%" PRId64, oldVer, newVer);
SRetrieveAnalAlgoReq req = {.dnodeId = pMgmt->pData->dnodeId, .analVer = oldVer};
int32_t contLen = tSerializeRetrieveAnalAlgoReq(NULL, 0, &req);
if (contLen < 0) {
dError("failed to serialize analysis function ver request since %s", tstrerror(contLen));
return;
}
void *pHead = rpcMallocCont(contLen);
contLen = tSerializeRetrieveAnalAlgoReq(pHead, contLen, &req);
if (contLen < 0) {
rpcFreeCont(pHead);
dError("failed to serialize analysis function ver request since %s", tstrerror(contLen));
return;
}
SRpcMsg rpcMsg = {
.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_RETRIEVE_ANAL_ALGO,
.info.ahandle = (void *)0x9527,
.info.refId = 0,
.info.noResp = 0,
.info.handle = 0,
};
SEpSet epset = {0};
(void)dmGetMnodeEpSet(pMgmt->pData, &epset);
code = rpcSendRequest(pMgmt->msgCb.clientRpc, &epset, &rpcMsg, NULL);
if (code != 0) {
dError("failed to send retrieve analysis func ver request since %s", tstrerror(code));
}
}
static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
const STraceId *trace = &pRsp->info.traceId;
dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code);
@ -111,6 +154,7 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
}
dmMayShouldUpdateIpWhiteList(pMgmt, statusRsp.ipWhiteVer);
dmMayShouldUpdateAnalFunc(pMgmt, statusRsp.analVer);
}
tFreeSStatusRsp(&statusRsp);
}
@ -172,6 +216,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
pMgmt->statusSeq++;
req.statusSeq = pMgmt->statusSeq;
req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
req.analVer = taosAnalGetVersion();
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
if (contLen < 0) {

7
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "dmInt.h"
#include "libs/function/tudf.h"
#include "tanal.h"
static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
int32_t code = 0;
@ -77,7 +78,11 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
}
if ((code = udfStartUdfd(pMgmt->pData->dnodeId)) != 0) {
dError("failed to start udfd");
dError("failed to start udfd since %s", tstrerror(code));
}
if ((code = taosAnalInit()) != 0) {
dError("failed to init analysis env since %s", tstrerror(code));
}
pOutput->pMgmt = pMgmt;

4
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -141,6 +141,9 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ANODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_UPDATE_ANODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ANODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -180,6 +183,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_ANAL_ALGO, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -20,6 +20,7 @@
#include "libs/function/tudf.h"
#include "tgrant.h"
#include "tcompare.h"
#include "tanal.h"
// clang-format on
#define DM_INIT_AUDIT() \
@ -214,6 +215,7 @@ void dmCleanup() {
dError("failed to close udfc");
}
udfStopUdfd();
taosAnalCleanup();
taosStopCacheRefreshWorker();
(void)dmDiskClose();
DestroyRegexCache();

2
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -65,7 +65,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = taosStrdup(path);
if (pWrapper->path == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _OVER;
}

18
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -17,6 +17,7 @@
#include "dmMgmt.h"
#include "qworker.h"
#include "tversion.h"
#include "tanal.h"
static inline void dmSendRsp(SRpcMsg *pMsg) {
if (rpcSendResponse(pMsg) != 0) {
@ -105,6 +106,17 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) {
return false;
}
}
static void dmUpdateAnalFunc(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
SRetrieveAnalAlgoRsp rsp = {0};
if (tDeserializeRetrieveAnalAlgoRsp(pRpc->pCont, pRpc->contLen, &rsp) == 0) {
taosAnalUpdate(rsp.ver, rsp.hash);
rsp.hash = NULL;
}
tFreeRetrieveAnalAlgoRsp(&rsp);
rpcFreeCont(pRpc->pCont);
}
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
SDnodeTrans *pTrans = &pDnode->trans;
int32_t code = -1;
@ -150,10 +162,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
dmSetMnodeEpSet(&pDnode->data, pEpSet);
}
break;
case TDMT_MND_RETRIEVE_IP_WHITE_RSP: {
case TDMT_MND_RETRIEVE_IP_WHITE_RSP:
dmUpdateRpcIpWhite(&pDnode->data, pTrans->serverRpc, pRpc);
return;
} break;
case TDMT_MND_RETRIEVE_ANAL_ALGO_RSP:
dmUpdateAnalFunc(&pDnode->data, pTrans->serverRpc, pRpc);
return;
default:
break;
}

32
source/dnode/mnode/impl/inc/mndAnode.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_ANODE_H_
#define _TD_MND_ANODE_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitAnode(SMnode *pMnode);
void mndCleanupAnode(SMnode *pMnode);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_ANODE_H_*/

21
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -78,6 +78,9 @@ typedef enum {
MND_OPER_DROP_VIEW,
MND_OPER_CONFIG_CLUSTER,
MND_OPER_BALANCE_VGROUP_LEADER,
MND_OPER_CREATE_ANODE,
MND_OPER_UPDATE_ANODE,
MND_OPER_DROP_ANODE
} EOperType;
typedef enum {
@ -232,6 +235,24 @@ typedef struct {
char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeObj;
typedef struct {
int32_t nameLen;
char* name;
} SAnodeAlgo;
typedef struct {
int32_t id;
int64_t createdTime;
int64_t updateTime;
int32_t version;
int32_t urlLen;
int32_t numOfAlgos;
int32_t status;
SRWLatch lock;
char* url;
SArray** algos;
} SAnodeObj;
typedef struct {
int32_t id;
int64_t createdTime;

902
source/dnode/mnode/impl/src/mndAnode.c Normal file
View File

@ -0,0 +1,902 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndAnode.h"
#include "audit.h"
#include "mndDnode.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "tanal.h"
#include "tjson.h"
#ifdef USE_ANAL
#define TSDB_ANODE_VER_NUMBER 1
#define TSDB_ANODE_RESERVE_SIZE 64
static SSdbRaw *mndAnodeActionEncode(SAnodeObj *pObj);
static SSdbRow *mndAnodeActionDecode(SSdbRaw *pRaw);
static int32_t mndAnodeActionInsert(SSdb *pSdb, SAnodeObj *pObj);
static int32_t mndAnodeActionUpdate(SSdb *pSdb, SAnodeObj *pOld, SAnodeObj *pNew);
static int32_t mndAnodeActionDelete(SSdb *pSdb, SAnodeObj *pObj);
static int32_t mndProcessCreateAnodeReq(SRpcMsg *pReq);
static int32_t mndProcessUpdateAnodeReq(SRpcMsg *pReq);
static int32_t mndProcessDropAnodeReq(SRpcMsg *pReq);
static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq);
static int32_t mndRetrieveAnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextAnode(SMnode *pMnode, void *pIter);
static int32_t mndRetrieveAnodesFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextAnodeFull(SMnode *pMnode, void *pIter);
static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj);
static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status);
int32_t mndInitAnode(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_ANODE,
.keyType = SDB_KEY_INT32,
.encodeFp = (SdbEncodeFp)mndAnodeActionEncode,
.decodeFp = (SdbDecodeFp)mndAnodeActionDecode,
.insertFp = (SdbInsertFp)mndAnodeActionInsert,
.updateFp = (SdbUpdateFp)mndAnodeActionUpdate,
.deleteFp = (SdbDeleteFp)mndAnodeActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ANODE, mndProcessCreateAnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_UPDATE_ANODE, mndProcessUpdateAnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_ANODE, mndProcessDropAnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_ANAL_ALGO, mndProcessAnalAlgoReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE, mndRetrieveAnodes);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ANODE, mndCancelGetNextAnode);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE_FULL, mndRetrieveAnodesFull);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ANODE_FULL, mndCancelGetNextAnodeFull);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupAnode(SMnode *pMnode) {}
SAnodeObj *mndAcquireAnode(SMnode *pMnode, int32_t anodeId) {
SAnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_ANODE, &anodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_ANODE_NOT_EXIST;
}
return pObj;
}
void mndReleaseAnode(SMnode *pMnode, SAnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pObj);
}
static SSdbRaw *mndAnodeActionEncode(SAnodeObj *pObj) {
int32_t code = 0;
int32_t lino = 0;
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t rawDataLen = sizeof(SAnodeObj) + TSDB_ANODE_RESERVE_SIZE + pObj->urlLen;
for (int32_t t = 0; t < pObj->numOfAlgos; ++t) {
SArray *algos = pObj->algos[t];
for (int32_t a = 0; a < (int32_t)taosArrayGetSize(algos); ++a) {
SAnodeAlgo *algo = taosArrayGet(algos, a);
rawDataLen += (2 * sizeof(int32_t) + algo->nameLen);
}
rawDataLen += sizeof(int32_t);
}
SSdbRaw *pRaw = sdbAllocRaw(SDB_ANODE, TSDB_ANODE_VER_NUMBER, rawDataLen);
if (pRaw == NULL) goto _OVER;
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER)
SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER)
SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER)
SDB_SET_INT32(pRaw, dataPos, pObj->version, _OVER)
SDB_SET_INT32(pRaw, dataPos, pObj->urlLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pObj->url, pObj->urlLen, _OVER)
SDB_SET_INT32(pRaw, dataPos, pObj->numOfAlgos, _OVER)
for (int32_t i = 0; i < pObj->numOfAlgos; ++i) {
SArray *algos = pObj->algos[i];
SDB_SET_INT32(pRaw, dataPos, (int32_t)taosArrayGetSize(algos), _OVER)
for (int32_t j = 0; j < (int32_t)taosArrayGetSize(algos); ++j) {
SAnodeAlgo *algo = taosArrayGet(algos, j);
SDB_SET_INT32(pRaw, dataPos, algo->nameLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, algo->name, algo->nameLen, _OVER)
SDB_SET_INT32(pRaw, dataPos, 0, _OVER) // reserved
}
}
SDB_SET_RESERVE(pRaw, dataPos, TSDB_ANODE_RESERVE_SIZE, _OVER)
terrno = 0;
_OVER:
if (terrno != 0) {
mError("anode:%d, failed to encode to raw:%p since %s", pObj->id, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("anode:%d, encode to raw:%p, row:%p", pObj->id, pRaw, pObj);
return pRaw;
}
static SSdbRow *mndAnodeActionDecode(SSdbRaw *pRaw) {
int32_t code = 0;
int32_t lino = 0;
terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SAnodeObj *pObj = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != TSDB_ANODE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER;
}
pRow = sdbAllocRow(sizeof(SAnodeObj));
if (pRow == NULL) goto _OVER;
pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pObj->version, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pObj->urlLen, _OVER)
if (pObj->urlLen > 0) {
pObj->url = taosMemoryCalloc(pObj->urlLen, 1);
if (pObj->url == NULL) goto _OVER;
SDB_GET_BINARY(pRaw, dataPos, pObj->url, pObj->urlLen, _OVER)
}
SDB_GET_INT32(pRaw, dataPos, &pObj->numOfAlgos, _OVER)
if (pObj->numOfAlgos > 0) {
pObj->algos = taosMemoryCalloc(pObj->numOfAlgos, sizeof(SArray *));
if (pObj->algos == NULL) {
goto _OVER;
}
}
for (int32_t i = 0; i < pObj->numOfAlgos; ++i) {
int32_t numOfAlgos = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfAlgos, _OVER)
pObj->algos[i] = taosArrayInit(2, sizeof(SAnodeAlgo));
if (pObj->algos[i] == NULL) goto _OVER;
for (int32_t j = 0; j < numOfAlgos; ++j) {
SAnodeAlgo algoObj = {0};
int32_t reserved = 0;
SDB_GET_INT32(pRaw, dataPos, &algoObj.nameLen, _OVER)
if (algoObj.nameLen > 0) {
algoObj.name = taosMemoryCalloc(algoObj.nameLen, 1);
if (algoObj.name == NULL) goto _OVER;
}
SDB_GET_BINARY(pRaw, dataPos, algoObj.name, algoObj.nameLen, _OVER)
SDB_GET_INT32(pRaw, dataPos, &reserved, _OVER);
if (taosArrayPush(pObj->algos[i], &algoObj) == NULL) goto _OVER;
}
}
SDB_GET_RESERVE(pRaw, dataPos, TSDB_ANODE_RESERVE_SIZE, _OVER)
terrno = 0;
_OVER:
if (terrno != 0) {
mError("anode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
if (pObj != NULL) {
taosMemoryFreeClear(pObj->url);
}
taosMemoryFreeClear(pRow);
return NULL;
}
mTrace("anode:%d, decode from raw:%p, row:%p", pObj->id, pRaw, pObj);
return pRow;
}
static void mndFreeAnode(SAnodeObj *pObj) {
taosMemoryFreeClear(pObj->url);
for (int32_t i = 0; i < pObj->numOfAlgos; ++i) {
SArray *algos = pObj->algos[i];
for (int32_t j = 0; j < (int32_t)taosArrayGetSize(algos); ++j) {
SAnodeAlgo *algo = taosArrayGet(algos, j);
taosMemoryFreeClear(algo->name);
}
taosArrayDestroy(algos);
}
taosMemoryFreeClear(pObj->algos);
}
static int32_t mndAnodeActionInsert(SSdb *pSdb, SAnodeObj *pObj) {
mTrace("anode:%d, perform insert action, row:%p", pObj->id, pObj);
return 0;
}
static int32_t mndAnodeActionDelete(SSdb *pSdb, SAnodeObj *pObj) {
mTrace("anode:%d, perform delete action, row:%p", pObj->id, pObj);
mndFreeAnode(pObj);
return 0;
}
static int32_t mndAnodeActionUpdate(SSdb *pSdb, SAnodeObj *pOld, SAnodeObj *pNew) {
mTrace("anode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
taosWLockLatch(&pOld->lock);
int32_t numOfAlgos = pNew->numOfAlgos;
void *algos = pNew->algos;
pNew->numOfAlgos = pOld->numOfAlgos;
pNew->algos = pOld->algos;
pOld->numOfAlgos = numOfAlgos;
pOld->algos = algos;
pOld->updateTime = pNew->updateTime;
pOld->version = pNew->version;
taosWUnLockLatch(&pOld->lock);
return 0;
}
static int32_t mndSetCreateAnodeRedoLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndAnodeActionEncode(pObj);
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateAnodeUndoLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndAnodeActionEncode(pObj);
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetCreateAnodeCommitLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndAnodeActionEncode(pObj);
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndCreateAnode(SMnode *pMnode, SRpcMsg *pReq, SMCreateAnodeReq *pCreate) {
int32_t code = -1;
STrans *pTrans = NULL;
SAnodeObj anodeObj = {0};
anodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_ANODE);
anodeObj.createdTime = taosGetTimestampMs();
anodeObj.updateTime = anodeObj.createdTime;
anodeObj.version = 0;
anodeObj.urlLen = pCreate->urlLen;
if (anodeObj.urlLen > TSDB_ANAL_ANODE_URL_LEN) {
code = TSDB_CODE_MND_ANODE_TOO_LONG_URL;
goto _OVER;
}
anodeObj.url = taosMemoryCalloc(1, pCreate->urlLen);
if (anodeObj.url == NULL) goto _OVER;
(void)memcpy(anodeObj.url, pCreate->url, pCreate->urlLen);
code = mndGetAnodeAlgoList(anodeObj.url, &anodeObj);
if (code != 0) goto _OVER;
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-anode");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create anode:%s as anode:%d", pTrans->id, pCreate->url, anodeObj.id);
TAOS_CHECK_GOTO(mndSetCreateAnodeRedoLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateAnodeUndoLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateAnodeCommitLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndFreeAnode(&anodeObj);
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
static SAnodeObj *mndAcquireAnodeByURL(SMnode *pMnode, char *url) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SAnodeObj *pAnode = NULL;
pIter = sdbFetch(pSdb, SDB_ANODE, pIter, (void **)&pAnode);
if (pIter == NULL) break;
if (strcasecmp(url, pAnode->url) == 0) {
sdbCancelFetch(pSdb, pIter);
return pAnode;
}
sdbRelease(pSdb, pAnode);
}
terrno = TSDB_CODE_MND_ANODE_NOT_EXIST;
return NULL;
}
static int32_t mndProcessCreateAnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SMCreateAnodeReq createReq = {0};
TAOS_CHECK_GOTO(tDeserializeSMCreateAnodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("anode:%s, start to create", createReq.url);
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_ANODE), NULL, _OVER);
pObj = mndAcquireAnodeByURL(pMnode, createReq.url);
if (pObj != NULL) {
code = TSDB_CODE_MND_ANODE_ALREADY_EXIST;
goto _OVER;
}
code = mndCreateAnode(pMnode, pReq, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("anode:%s, failed to create since %s", createReq.url, tstrerror(code));
}
mndReleaseAnode(pMnode, pObj);
tFreeSMCreateAnodeReq(&createReq);
TAOS_RETURN(code);
}
static int32_t mndUpdateAnode(SMnode *pMnode, SAnodeObj *pAnode, SRpcMsg *pReq) {
mInfo("anode:%d, start to update", pAnode->id);
int32_t code = -1;
STrans *pTrans = NULL;
SAnodeObj anodeObj = {0};
anodeObj.id = pAnode->id;
anodeObj.updateTime = taosGetTimestampMs();
code = mndGetAnodeAlgoList(pAnode->url, &anodeObj);
if (code != 0) goto _OVER;
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-anode");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to update anode:%d", pTrans->id, anodeObj.id);
TAOS_CHECK_GOTO(mndSetCreateAnodeCommitLogs(pTrans, &anodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndFreeAnode(&anodeObj);
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
static int32_t mndUpdateAllAnodes(SMnode *pMnode, SRpcMsg *pReq) {
mInfo("update all anodes");
SSdb *pSdb = pMnode->pSdb;
int32_t code = 0;
int32_t rows = 0;
int32_t numOfRows = sdbGetSize(pSdb, SDB_ANODE);
void *pIter = NULL;
while (1) {
SAnodeObj *pObj = NULL;
ESdbStatus objStatus = 0;
pIter = sdbFetchAll(pSdb, SDB_ANODE, pIter, (void **)&pObj, &objStatus, true);
if (pIter == NULL) break;
rows++;
void *transReq = NULL;
if (rows == numOfRows) transReq = pReq;
code = mndUpdateAnode(pMnode, pObj, transReq);
sdbRelease(pSdb, pObj);
if (code != 0) break;
}
if (code == 0 && rows == numOfRows) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
}
return code;
}
static int32_t mndProcessUpdateAnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SMUpdateAnodeReq updateReq = {0};
TAOS_CHECK_GOTO(tDeserializeSMUpdateAnodeReq(pReq->pCont, pReq->contLen, &updateReq), NULL, _OVER);
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_UPDATE_ANODE), NULL, _OVER);
if (updateReq.anodeId == -1) {
code = mndUpdateAllAnodes(pMnode, pReq);
} else {
pObj = mndAcquireAnode(pMnode, updateReq.anodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_ANODE_NOT_EXIST;
goto _OVER;
}
code = mndUpdateAnode(pMnode, pObj, pReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
}
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
if (updateReq.anodeId != -1) {
mError("anode:%d, failed to update since %s", updateReq.anodeId, tstrerror(code));
}
}
mndReleaseAnode(pMnode, pObj);
tFreeSMUpdateAnodeReq(&updateReq);
TAOS_RETURN(code);
}
static int32_t mndSetDropAnodeRedoLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndAnodeActionEncode(pObj);
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropAnodeCommitLogs(STrans *pTrans, SAnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndAnodeActionEncode(pObj);
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetDropAnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SAnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
TAOS_CHECK_RETURN(mndSetDropAnodeRedoLogs(pTrans, pObj));
TAOS_CHECK_RETURN(mndSetDropAnodeCommitLogs(pTrans, pObj));
return 0;
}
static int32_t mndDropAnode(SMnode *pMnode, SRpcMsg *pReq, SAnodeObj *pObj) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-anode");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop anode:%d", pTrans->id, pObj->id);
TAOS_CHECK_GOTO(mndSetDropAnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
static int32_t mndProcessDropAnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SMDropAnodeReq dropReq = {0};
TAOS_CHECK_GOTO(tDeserializeSMDropAnodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("anode:%d, start to drop", dropReq.anodeId);
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_ANODE), NULL, _OVER);
if (dropReq.anodeId <= 0) {
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireAnode(pMnode, dropReq.anodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
code = mndDropAnode(pMnode, pReq, pObj);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("anode:%d, failed to drop since %s", dropReq.anodeId, tstrerror(code));
}
mndReleaseAnode(pMnode, pObj);
tFreeSMDropAnodeReq(&dropReq);
TAOS_RETURN(code);
}
static int32_t mndRetrieveAnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
int32_t cols = 0;
SAnodeObj *pObj = NULL;
char buf[TSDB_ANAL_ANODE_URL_LEN + VARSTR_HEADER_SIZE];
char status[64];
int32_t code = 0;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_ANODE, pShow->pIter, (void **)&pObj);
if (pShow->pIter == NULL) break;
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
if (code != 0) goto _end;
STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->url, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)buf, false);
if (code != 0) goto _end;
status[0] = 0;
if (mndGetAnodeStatus(pObj, status) == 0) {
STR_TO_VARSTR(buf, status);
} else {
STR_TO_VARSTR(buf, "offline");
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, buf, false);
if (code != 0) goto _end;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
if (code != 0) goto _end;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->updateTime, false);
if (code != 0) goto _end;
numOfRows++;
sdbRelease(pSdb, pObj);
}
_end:
if (code != 0) sdbRelease(pSdb, pObj);
pShow->numOfRows += numOfRows;
return numOfRows;
}
static void mndCancelGetNextAnode(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetchByType(pSdb, pIter, SDB_ANODE);
}
static int32_t mndRetrieveAnodesFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
int32_t cols = 0;
SAnodeObj *pObj = NULL;
char buf[TSDB_ANAL_ALGO_NAME_LEN + VARSTR_HEADER_SIZE];
int32_t code = 0;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_ANODE, pShow->pIter, (void **)&pObj);
if (pShow->pIter == NULL) break;
for (int32_t t = 0; t < pObj->numOfAlgos; ++t) {
SArray *algos = pObj->algos[t];
for (int32_t a = 0; a < taosArrayGetSize(algos); ++a) {
SAnodeAlgo *algo = taosArrayGet(algos, a);
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
if (code != 0) goto _end;
STR_TO_VARSTR(buf, taosAnalAlgoStr(t));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, buf, false);
if (code != 0) goto _end;
STR_TO_VARSTR(buf, algo->name);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
code = colDataSetVal(pColInfo, numOfRows, buf, false);
if (code != 0) goto _end;
numOfRows++;
}
}
sdbRelease(pSdb, pObj);
}
_end:
if (code != 0) sdbRelease(pSdb, pObj);
pShow->numOfRows += numOfRows;
return numOfRows;
}
static void mndCancelGetNextAnodeFull(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetchByType(pSdb, pIter, SDB_ANODE);
}
static int32_t mndDecodeAlgoList(SJson *pJson, SAnodeObj *pObj) {
int32_t code = 0;
int32_t protocol = 0;
double tmp = 0;
char buf[TSDB_ANAL_ALGO_NAME_LEN + 1] = {0};
code = tjsonGetDoubleValue(pJson, "protocol", &tmp);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
protocol = (int32_t)(tmp * 1000);
if (protocol != 100) return TSDB_CODE_MND_ANODE_INVALID_PROTOCOL;
code = tjsonGetDoubleValue(pJson, "version", &tmp);
pObj->version = (int32_t)(tmp * 1000);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
if (pObj->version <= 0) return TSDB_CODE_MND_ANODE_INVALID_VERSION;
SJson *details = tjsonGetObjectItem(pJson, "details");
if (details == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t numOfDetails = tjsonGetArraySize(details);
pObj->algos = taosMemoryCalloc(ANAL_ALGO_TYPE_END, sizeof(SArray *));
if (pObj->algos == NULL) return TSDB_CODE_OUT_OF_MEMORY;
pObj->numOfAlgos = ANAL_ALGO_TYPE_END;
for (int32_t i = 0; i < ANAL_ALGO_TYPE_END; ++i) {
pObj->algos[i] = taosArrayInit(4, sizeof(SAnodeAlgo));
if (pObj->algos[i] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t d = 0; d < numOfDetails; ++d) {
SJson *detail = tjsonGetArrayItem(details, d);
if (detail == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
code = tjsonGetStringValue(detail, "type", buf);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
EAnalAlgoType type = taosAnalAlgoInt(buf);
if (type < 0 || type >= ANAL_ALGO_TYPE_END) continue;
SJson *algos = tjsonGetObjectItem(detail, "algo");
if (algos == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t numOfAlgos = tjsonGetArraySize(algos);
for (int32_t a = 0; a < numOfAlgos; ++a) {
SJson *algo = tjsonGetArrayItem(algos, a);
if (algo == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
code = tjsonGetStringValue(algo, "name", buf);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
SAnodeAlgo algoObj = {0};
algoObj.nameLen = strlen(buf) + 1;
if (algoObj.nameLen > TSDB_ANAL_ALGO_NAME_LEN) return TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME;
if (algoObj.nameLen <= 1) return TSDB_CODE_OUT_OF_MEMORY;
algoObj.name = taosMemoryCalloc(algoObj.nameLen, 1);
tstrncpy(algoObj.name, buf, algoObj.nameLen);
if (taosArrayPush(pObj->algos[type], &algoObj) == NULL) return TSDB_CODE_OUT_OF_MEMORY;
}
}
return 0;
}
static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj) {
char anodeUrl[TSDB_ANAL_ANODE_URL_LEN + 1] = {0};
snprintf(anodeUrl, TSDB_ANAL_ANODE_URL_LEN, "%s/%s", url, "list");
SJson *pJson = taosAnalSendReqRetJson(anodeUrl, ANAL_HTTP_TYPE_GET, NULL);
if (pJson == NULL) return terrno;
int32_t code = mndDecodeAlgoList(pJson, pObj);
if (pJson != NULL) tjsonDelete(pJson);
TAOS_RETURN(code);
}
static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status) {
int32_t code = 0;
int32_t protocol = 0;
double tmp = 0;
char anodeUrl[TSDB_ANAL_ANODE_URL_LEN + 1] = {0};
snprintf(anodeUrl, TSDB_ANAL_ANODE_URL_LEN, "%s/%s", pObj->url, "status");
SJson *pJson = taosAnalSendReqRetJson(anodeUrl, ANAL_HTTP_TYPE_GET, NULL);
if (pJson == NULL) return terrno;
code = tjsonGetDoubleValue(pJson, "protocol", &tmp);
if (code < 0) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
protocol = (int32_t)(tmp * 1000);
if (protocol != 100) {
code = TSDB_CODE_MND_ANODE_INVALID_PROTOCOL;
goto _OVER;
}
code = tjsonGetStringValue(pJson, "status", status);
if (code < 0) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (strlen(status) == 0) {
code = TSDB_CODE_MND_ANODE_INVALID_PROTOCOL;
goto _OVER;
}
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
TAOS_RETURN(code);
}
static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t code = -1;
SAnodeObj *pObj = NULL;
SAnalUrl url;
int32_t nameLen;
char name[TSDB_ANAL_ALGO_KEY_LEN];
SRetrieveAnalAlgoReq req = {0};
SRetrieveAnalAlgoRsp rsp = {0};
TAOS_CHECK_GOTO(tDeserializeRetrieveAnalAlgoReq(pReq->pCont, pReq->contLen, &req), NULL, _OVER);
rsp.ver = sdbGetTableVer(pSdb, SDB_ANODE);
if (req.analVer != rsp.ver) {
mInfo("dnode:%d, update analysis old ver:%" PRId64 " to new ver:%" PRId64, req.dnodeId, req.analVer, rsp.ver);
rsp.hash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (rsp.hash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
void *pIter = NULL;
while (1) {
SAnodeObj *pAnode = NULL;
pIter = sdbFetch(pSdb, SDB_ANODE, pIter, (void **)&pAnode);
if (pIter == NULL) break;
url.anode = pAnode->id;
for (int32_t t = 0; t < pAnode->numOfAlgos; ++t) {
SArray *algos = pAnode->algos[t];
url.type = t;
for (int32_t a = 0; a < taosArrayGetSize(algos); ++a) {
SAnodeAlgo *algo = taosArrayGet(algos, a);
nameLen = 1 + snprintf(name, sizeof(name) - 1, "%d:%s", url.type, algo->name);
SAnalUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen);
if (pOldUrl == NULL || (pOldUrl != NULL && pOldUrl->anode < url.anode)) {
if (pOldUrl != NULL) {
taosMemoryFreeClear(pOldUrl->url);
if (taosHashRemove(rsp.hash, name, nameLen) != 0) {
sdbRelease(pSdb, pAnode);
goto _OVER;
}
}
url.url = taosMemoryMalloc(TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN + 1);
if (url.url == NULL) {
sdbRelease(pSdb, pAnode);
goto _OVER;
}
url.urlLen = 1 + snprintf(url.url, TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN, "%s/%s", pAnode->url,
taosAnalAlgoUrlStr(url.type));
if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalUrl)) != 0) {
taosMemoryFree(url.url);
sdbRelease(pSdb, pAnode);
goto _OVER;
}
}
}
sdbRelease(pSdb, pAnode);
}
}
}
int32_t contLen = tSerializeRetrieveAnalAlgoRsp(NULL, 0, &rsp);
void *pHead = rpcMallocCont(contLen);
(void)tSerializeRetrieveAnalAlgoRsp(pHead, contLen, &rsp);
pReq->info.rspLen = contLen;
pReq->info.rsp = pHead;
_OVER:
tFreeRetrieveAnalAlgoRsp(&rsp);
TAOS_RETURN(code);
}
#else
static int32_t mndProcessUnsupportReq(SRpcMsg *pReq) { return TSDB_CODE_OPS_NOT_SUPPORT; }
static int32_t mndRetrieveUnsupport(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
int32_t mndInitAnode(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ANODE, mndProcessUnsupportReq);
mndSetMsgHandle(pMnode, TDMT_MND_UPDATE_ANODE, mndProcessUnsupportReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_ANODE, mndProcessUnsupportReq);
mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_ANAL_ALGO, mndProcessUnsupportReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE, mndRetrieveUnsupport);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ANODE_FULL, mndRetrieveUnsupport);
return 0;
}
void mndCleanupAnode(SMnode *pMnode) {}
#endif

8
source/dnode/mnode/impl/src/mndDb.c
View File

@ -2366,7 +2366,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER);
char durationVstr[128] = {0};
int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], pDb->cfg.daysPerFile);
int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, pDb->cfg.daysPerFile);
varDataSetLen(durationVstr, len);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -2377,9 +2377,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
char keep1Str[128] = {0};
char keep2Str[128] = {0};
int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0);
int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1);
int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2);
int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pDb->cfg.daysToKeep0);
int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pDb->cfg.daysToKeep1);
int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pDb->cfg.daysToKeep2);
if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str);

8
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -693,7 +693,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
int64_t clusterid = mndGetClusterId(pMnode);
if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) {
code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER;
mWarn("dnode:%d, %s, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 ", code:0x%x",
mWarn("dnode:%d, %s, its clusterid:%" PRId64 " differ from current clusterid:%" PRId64 ", code:0x%x",
statusReq.dnodeId, statusReq.dnodeEp, statusReq.clusterId, clusterid, code);
goto _OVER;
}
@ -730,6 +730,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pMnode->ipWhiteVer = mndGetIpWhiteVer(pMnode);
int64_t analVer = sdbGetTableVer(pMnode->pSdb, SDB_ANODE);
int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
int64_t curMs = taosGetTimestampMs();
bool online = mndIsDnodeOnline(pDnode, curMs);
@ -738,9 +739,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum;
bool enableWhiteListChanged = statusReq.clusterCfg.enableWhiteList != (tsEnableWhiteList ? 1 : 0);
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged ||
bool analVerChanged = (analVer != statusReq.analVer);
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged || analVerChanged ||
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged || enableWhiteListChanged;
const STraceId *trace = &pReq->info.traceId;
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq);
@ -863,6 +864,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SStatusRsp statusRsp = {0};
statusRsp.statusSeq++;
statusRsp.analVer = analVer;
statusRsp.dnodeVer = dnodeVer;
statusRsp.dnodeCfg.dnodeId = pDnode->id;
statusRsp.dnodeCfg.clusterId = pMnode->clusterId;

2
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -662,6 +662,8 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->pTags = NULL;
pNew->pColumns = NULL;
pNew->pCmpr = NULL;
pNew->pTags = NULL;
pNew->updateTime = taosGetTimestampMs();
pNew->lock = 0;

4
source/dnode/mnode/impl/src/mndMain.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "mndAcct.h"
#include "mndArbGroup.h"
#include "mndAnode.h"
#include "mndCluster.h"
#include "mndCompact.h"
#include "mndCompactDetail.h"
@ -495,7 +496,7 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
int32_t code = 0;
pMnode->path = taosStrdup(path);
if (pMnode->path == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
TAOS_RETURN(code);
}
@ -605,6 +606,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-anode", mndInitAnode, mndCleanupAnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser));

4
source/dnode/mnode/impl/src/mndShow.c
View File

@ -68,6 +68,10 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_QNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
type = TSDB_MGMT_TABLE_SNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_ANODES, len) == 0) {
type = TSDB_MGMT_TABLE_ANODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_ANODES_FULL, len) == 0) {
type = TSDB_MGMT_TABLE_ANODE_FULL;
} else if (strncasecmp(name, TSDB_INS_TABLE_ARBGROUPS, len) == 0) {
type = TSDB_MGMT_TABLE_ARBGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) {

2
source/dnode/mnode/impl/src/mndSma.c
View File

@ -2350,7 +2350,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa
nodesDestroyNode(pNode);
}
pInfo->ast = taosStrdup(pSma->ast);
if (!pInfo->ast) code = TSDB_CODE_OUT_OF_MEMORY;
if (!pInfo->ast) code = terrno;
if (code == TSDB_CODE_SUCCESS && pDestStb->numOfTags > 0) {
pInfo->pTags = taosArrayInit(pDestStb->numOfTags, sizeof(SSchema));

1
source/dnode/mnode/impl/src/mndStreamTransAct.c
View File

@ -234,6 +234,7 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) {
mError("failed to extract epset during create update epset, code:%s", tstrerror(code));
taosMemoryFree(pBuf);
return code;
}

5
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -1495,7 +1495,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr
return code;
}
mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->actionPos);
mInfo("trans:%d, execute %d actions serial, current action:%d", pTrans->id, numOfActions, pTrans->actionPos);
for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pActions, action);
@ -1768,7 +1768,8 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool to
if (code == 0) {
pTrans->stage = TRN_STAGE_UNDO_ACTION;
mInfo("trans:%d, stage from rollback to undoAction", pTrans->id);
pTrans->actionPos = 0;
mInfo("trans:%d, stage from rollback to undoAction, actionPos:%d", pTrans->id, pTrans->actionPos);
continueExec = true;
} else {
pTrans->failedTimes++;

4
source/dnode/mnode/impl/src/mndUser.c
View File

@ -594,7 +594,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) {
if (name == NULL) {
sdbRelease(pSdb, pUser);
sdbCancelFetch(pSdb, pIter);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER);
TAOS_CHECK_GOTO(terrno, &lino, _OVER);
}
if (taosArrayPush(pUserNames, &name) == NULL) {
taosMemoryFree(name);
@ -617,7 +617,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) {
if (found == false) {
char *name = taosStrdup(TSDB_DEFAULT_USER);
if (name == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER);
TAOS_CHECK_GOTO(terrno, &lino, _OVER);
}
if (taosArrayPush(pUserNames, &name) == NULL) {
taosMemoryFree(name);

3
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -161,7 +161,8 @@ typedef enum {
SDB_COMPACT_DETAIL = 25,
SDB_GRANT = 26, // grant log
SDB_ARBGROUP = 27,
SDB_MAX = 28
SDB_ANODE = 28,
SDB_MAX = 29
} ESdbType;
typedef struct SSdbRaw {

57
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -25,6 +25,9 @@
#define SDB_RESERVE_SIZE 512
#define SDB_FILE_VER 1
#define SDB_TABLE_SIZE_EXTRA SDB_MAX
#define SDB_RESERVE_SIZE_EXTRA (512 - (SDB_TABLE_SIZE_EXTRA - SDB_TABLE_SIZE) * 2 * sizeof(int64_t))
static int32_t sdbDeployData(SSdb *pSdb) {
int32_t code = 0;
mInfo("start to deploy sdb");
@ -154,7 +157,38 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
// for sdb compatibility
for (int32_t i = SDB_TABLE_SIZE; i < SDB_TABLE_SIZE_EXTRA; ++i) {
int64_t maxId = 0;
ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (i < SDB_MAX) {
pSdb->maxId[i] = maxId;
}
int64_t ver = 0;
ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (i < SDB_MAX) {
pSdb->tableVer[i] = ver;
}
}
char reserve[SDB_RESERVE_SIZE_EXTRA] = {0};
ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) {
return terrno;
@ -205,7 +239,26 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
// for sdb compatibility
for (int32_t i = SDB_TABLE_SIZE; i < SDB_TABLE_SIZE_EXTRA; ++i) {
int64_t maxId = 0;
if (i < SDB_MAX) {
maxId = pSdb->maxId[i];
}
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
return terrno;
}
int64_t ver = 0;
if (i < SDB_MAX) {
ver = pSdb->tableVer[i];
}
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
return terrno;
}
}
char reserve[SDB_RESERVE_SIZE_EXTRA] = {0};
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
return terrno;
}

2
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -74,6 +74,8 @@ const char *sdbTableName(ESdbType type) {
return "grant";
case SDB_ARBGROUP:
return "arb_group";
case SDB_ANODE:
return "anode";
default:
return "undefine";
}

2
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -596,7 +596,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void
pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]);
if (pReader->status.pPrimaryTsCol == NULL) {
code = TSDB_CODE_INVALID_PARA;
code = terrno;
goto _end;
}

17
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -623,6 +623,7 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) {
int32_t lino = 0;
if (writer->ctx->fset) {
#if 0
// open data reader
SDataFileReaderConfig dataFileReaderConfig = {
.tsdb = writer->tsdb,
@ -650,6 +651,7 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) {
code = tsdbDataFileReaderOpen(NULL, &dataFileReaderConfig, &writer->ctx->dataReader);
TSDB_CHECK_CODE(code, lino, _exit);
#endif
// open stt reader array
SSttLvl* lvl;
@ -791,6 +793,15 @@ static int32_t tsdbSnapWriteFileSetOpenWriter(STsdbSnapWriter* writer) {
.did = writer->ctx->did,
.level = 0,
};
// merge stt files to either data or a new stt file
if (writer->ctx->fset) {
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {
if (writer->ctx->fset->farr[ftype] != NULL) {
config.files[ftype].exist = true;
config.files[ftype].file = writer->ctx->fset->farr[ftype]->f[0];
}
}
}
code = tsdbFSetWriterOpen(&config, &writer->ctx->fsetWriter);
TSDB_CHECK_CODE(code, lino, _exit);
@ -842,6 +853,8 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) {
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
} else {
tsdbInfo("vgId:%d %s succeeded, fid:%d", TD_VID(writer->tsdb->pVnode), __func__, fid);
}
return code;
}
@ -922,6 +935,8 @@ static int32_t tsdbSnapWriteFileSetEnd(STsdbSnapWriter* writer) {
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
} else {
tsdbInfo("vgId:%d %s succeeded, fid:%d", TD_VID(writer->tsdb->pVnode), __func__, writer->ctx->fid);
}
return code;
}
@ -1175,7 +1190,7 @@ _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
} else {
tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
tsdbInfo("vgId:%d %s done, rollback:%d", TD_VID(tsdb->pVnode), __func__, rollback);
}
return code;
}

5
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -102,9 +102,8 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) {
ts.tv_sec = tv.tv_sec;
}
int32_t rc = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts);
if (rc && rc != ETIMEDOUT) {
code = TAOS_SYSTEM_ERROR(rc);
code = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts);
if (code && code != TSDB_CODE_TIMEOUT_ERROR) {
TSDB_CHECK_CODE(code, lino, _exit);
}
}

6
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -254,7 +254,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
if (mer1.me.ctbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
}
@ -273,7 +273,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
if (mer1.me.ntbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
}
@ -399,7 +399,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
for (int32_t i = 0; i < msgNum; ++i) {
req = taosArrayGet(batchReq.pMsgs, i);
if (req == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}

10
source/libs/catalog/src/ctgUtil.c
View File

@ -1108,7 +1108,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ
if (target) {
pCtx->target = taosStrdup(target);
if (NULL == pCtx->target) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_RET(terrno);
}
} else {
pCtx->target = NULL;
@ -1125,7 +1125,7 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) {
if (target) {
ctx.target = taosStrdup(target);
if (NULL == ctx.target) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_RET(terrno);
}
}
@ -1631,7 +1631,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
if (NULL == (*dst)->vgArray) {
taosHashCleanup((*dst)->vgHash);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_RET(terrno);
}
}
@ -1698,7 +1698,7 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) {
}
pInfo->expr = taosStrdup(pInfo->expr);
if (NULL == pInfo->expr) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_RET(terrno);
}
}
@ -1712,7 +1712,7 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, cha
pMsgSendInfo->target.vgId = vgId;
pMsgSendInfo->target.dbFName = taosStrdup(dbFName);
if (NULL == pMsgSendInfo->target.dbFName) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_RET(terrno);
}
} else {
pMsgSendInfo->target.type = TARGET_TYPE_MNODE;

174
source/libs/command/src/command.c
View File

@ -291,7 +291,8 @@ static int32_t buildRetension(SArray* pRetension, char** ppRetentions) {
return TSDB_CODE_SUCCESS;
}
char* p1 = taosMemoryCalloc(1, 100);
const int lMaxLen = 128;
char* p1 = taosMemoryCalloc(1, lMaxLen);
if (NULL == p1) {
return terrno;
}
@ -302,13 +303,13 @@ static int32_t buildRetension(SArray* pRetension, char** ppRetentions) {
int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
if (i == 0) {
len += sprintf(p1 + len, "-:%" PRId64 "%c", v2, p->keepUnit);
len += snprintf(p1 + len, lMaxLen - len, "-:%" PRId64 "%c", v2, p->keepUnit);
} else {
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
len += snprintf(p1 + len, lMaxLen - len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
}
if (i < size - 1) {
len += sprintf(p1 + len, ",");
len += snprintf(p1 + len, lMaxLen - len, ",");
}
}
@ -344,16 +345,19 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) {
return TSDB_CACHE_MODEL_NONE_STR;
}
int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes) {
int32_t formatDurationOrKeep(char* buffer, int64_t bufSize, int32_t timeInMinutes) {
if (buffer == NULL || bufSize <= 0) {
return 0;
}
int32_t len = 0;
if (timeInMinutes % 1440 == 0) {
int32_t days = timeInMinutes / 1440;
len = sprintf(buffer, "%dd", days);
len = snprintf(buffer, bufSize, "%dd", days);
} else if (timeInMinutes % 60 == 0) {
int32_t hours = timeInMinutes / 60;
len = sprintf(buffer, "%dh", hours);
len = snprintf(buffer, bufSize, "%dh", hours);
} else {
len = sprintf(buffer, "%dm", timeInMinutes);
len = snprintf(buffer, bufSize, "%dm", timeInMinutes);
}
return len;
}
@ -400,15 +404,15 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName,
char keep1Str[128] = {0};
char keep2Str[128] = {0};
int32_t lenDuration = formatDurationOrKeep(durationStr, pCfg->daysPerFile);
int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pCfg->daysToKeep0);
int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pCfg->daysToKeep1);
int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pCfg->daysToKeep2);
int32_t lenDuration = formatDurationOrKeep(durationStr, sizeof(durationStr), pCfg->daysPerFile);
int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pCfg->daysToKeep0);
int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pCfg->daysToKeep1);
int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2);
if (IS_SYS_DBNAME(dbName)) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName);
len += snprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName);
} else {
len += sprintf(buf2 + VARSTR_HEADER_SIZE,
len += snprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %s "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %s,%s,%s PAGES %d PAGESIZE %d "
"PRECISION '%s' REPLICA %d "
@ -426,7 +430,7 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName,
pCfg->s3KeepLocal, pCfg->s3Compact);
if (pRetentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", pRetentions);
len += snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, " RETENTIONS %s", pRetentions);
}
}
@ -503,28 +507,32 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) {
void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfColumns; ++i) {
SSchema* pSchema = pCfg->pSchemas + i;
char type[32 + 60]; // 60 byte for compress info
sprintf(type, "%s", tDataTypes[pSchema->type].name);
#define LTYPE_LEN (32 + 60) // 60 byte for compress info
char type[LTYPE_LEN];
snprintf(type, LTYPE_LEN, "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type ||
TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
snprintf(type + strlen(type), LTYPE_LEN - strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
snprintf(type + strlen(type), LTYPE_LEN - strlen(type), "(%d)",
(int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
}
if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) {
sprintf(type + strlen(type), " ENCODE \'%s\'",
snprintf(type + strlen(type), LTYPE_LEN - strlen(type), " ENCODE \'%s\'",
columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress)));
sprintf(type + strlen(type), " COMPRESS \'%s\'",
snprintf(type + strlen(type), LTYPE_LEN - strlen(type), " COMPRESS \'%s\'",
columnCompressStr(COMPRESS_L2_TYPE_U32(pCfg->pSchemaExt[i].compress)));
sprintf(type + strlen(type), " LEVEL \'%s\'",
snprintf(type + strlen(type), LTYPE_LEN - strlen(type), " LEVEL \'%s\'",
columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress)));
}
if (!(pSchema->flags & COL_IS_KEY)) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s",
((i > 0) ? ", " : ""), pSchema->name, type);
} else {
char* pk = "PRIMARY KEY";
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s %s", ((i > 0) ? ", " : ""), pSchema->name, type, pk);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s %s",
((i > 0) ? ", " : ""), pSchema->name, type, pk);
}
}
}
@ -533,22 +541,25 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
char type[32];
sprintf(type, "%s", tDataTypes[pSchema->type].name);
snprintf(type, sizeof(type), "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type ||
TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)",
(int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, sizeof(type) - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s",
((i > 0) ? ", " : ""), pSchema->name, type);
}
}
void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"%s`%s`", ((i > 0) ? ", " : ""), pSchema->name);
}
}
@ -569,7 +580,8 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
qError("failed to parse tag to json, pJson is NULL");
return terrno;
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"%s", pJson);
taosMemoryFree(pJson);
return TSDB_CODE_SUCCESS;
@ -582,11 +594,13 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
if (i > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", ");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
", ");
}
if (j >= valueNum) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"NULL");
continue;
}
@ -599,17 +613,24 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
char type = pTagVal->type;
int32_t tlen = 0;
int64_t leftSize = SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len);
if (leftSize <= 0) {
qError("no enough space to store tag value, leftSize:%" PRId64, leftSize);
code = TSDB_CODE_APP_ERROR;
TAOS_CHECK_ERRNO(code);
}
if (IS_VAR_DATA_TYPE(type)) {
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen);
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, pTagVal->pData, pTagVal->nData, &tlen);
TAOS_CHECK_ERRNO(code);
} else {
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen);
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen);
TAOS_CHECK_ERRNO(code);
}
*len += tlen;
j++;
} else {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"NULL");
}
}
_exit:
@ -620,37 +641,47 @@ _exit:
void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) {
if (pCfg->commentLen > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" COMMENT '%s'", pCfg->pComment);
} else if (0 == pCfg->commentLen) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT ''");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" COMMENT ''");
}
if (NULL != pDbCfg->pRetensions && pCfg->watermark1 > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" WATERMARK %" PRId64 "a", pCfg->watermark1);
if (pCfg->watermark2 > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
", %" PRId64 "a", pCfg->watermark2);
}
}
if (NULL != pDbCfg->pRetensions && pCfg->delay1 > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" MAX_DELAY %" PRId64 "a", pCfg->delay1);
if (pCfg->delay2 > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
", %" PRId64 "a", pCfg->delay2);
}
}
int32_t funcNum = taosArrayGetSize(pCfg->pFuncs);
if (NULL != pDbCfg->pRetensions && funcNum > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP(");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" ROLLUP(");
for (int32_t i = 0; i < funcNum; ++i) {
char* pFunc = taosArrayGet(pCfg->pFuncs, i);
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"%s%s", ((i > 0) ? ", " : ""), pFunc);
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")");
*len +=
snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ")");
}
if (pCfg->ttl > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" TTL %d", pCfg->ttl);
}
if (TSDB_SUPER_TABLE == pCfg->tableType || TSDB_NORMAL_TABLE == pCfg->tableType) {
@ -663,18 +694,23 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg*
if (nSma < pCfg->numOfColumns && nSma > 0) {
bool smaOn = false;
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " SMA(");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" SMA(");
for (int32_t i = 0; i < pCfg->numOfColumns; ++i) {
if (IS_BSMA_ON(pCfg->pSchemas + i)) {
if (smaOn) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ",`%s`", (pCfg->pSchemas + i)->name);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len,
SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ",`%s`",
(pCfg->pSchemas + i)->name);
} else {
smaOn = true;
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "`%s`", (pCfg->pSchemas + i)->name);
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len,
SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "`%s`",
(pCfg->pSchemas + i)->name);
}
}
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")");
*len += snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, ")");
}
}
}
@ -698,24 +734,32 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
int32_t len = 0;
if (TSDB_SUPER_TABLE == pCfg->tableType) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName);
len += snprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE STABLE `%s` (", tbName);
appendColumnFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS (");
len += snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len),
") TAGS (");
appendTagFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
len +=
snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")");
appendTableOptions(buf2, &len, pDbCfg, pCfg);
} else if (TSDB_CHILD_TABLE == pCfg->tableType) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName);
len += snprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName);
appendTagNameFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS (");
len += snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len),
") TAGS (");
code = appendTagValues(buf2, &len, pCfg);
TAOS_CHECK_ERRNO(code);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
len +=
snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")");
appendTableOptions(buf2, &len, pDbCfg, pCfg);
} else {
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName);
len += snprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE TABLE `%s` (", tbName);
appendColumnFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
len +=
snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")");
appendTableOptions(buf2, &len, pDbCfg, pCfg);
}
@ -792,9 +836,21 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) {
taosResetLog();
cfgDumpCfg(tsCfg, 0, false);
} else if (0 == strcasecmp(cmd, COMMAND_SCHEDULE_POLICY)) {
code = schedulerUpdatePolicy(atoi(value));
int32_t tmp = 0;
code = taosStr2int32(value, &tmp);
if (code) {
qError("invalid value:%s, error:%s", value, tstrerror(code));
return code;
}
code = schedulerUpdatePolicy(tmp);
} else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) {
code = schedulerEnableReSchedule(atoi(value));
int32_t tmp = 0;
code = taosStr2int32(value, &tmp);
if (code) {
qError("invalid value:%s, error:%s", value, tstrerror(code));
return code;
}
code = schedulerEnableReSchedule(tmp != 0);
} else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) {
code = ctgdHandleDbgCommand(value);
} else if (0 == strcasecmp(cmd, COMMAND_ENABLE_MEM_DEBUG)) {

3
source/libs/executor/inc/executorInt.h
View File

@ -833,6 +833,9 @@ void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExpr
SGroupResInfo* pGroupResInfo);
void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap);
void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo);
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey, void* pState, SFunctionStateStore* pStore);
void cleanupAggSup(SAggSupporter* pAggSup);

7
source/libs/executor/inc/operator.h
View File

@ -133,6 +133,8 @@ int32_t createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPart
int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SHashJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
@ -159,6 +161,8 @@ int32_t createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy
int32_t createGroupCacheOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SGroupCachePhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
// clang-format on
@ -190,6 +194,9 @@ int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdSt
int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList);
void * getOperatorParam(int32_t opType, SOperatorParam* param, int32_t idx);
void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId);
void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, uint64_t groupId);
#ifdef __cplusplus
}
#endif

40
source/libs/executor/src/aggregateoperator.c
View File

@ -159,8 +159,8 @@ void destroyAggOperatorInfo(void* param) {
cleanupBasicInfo(&pInfo->binfo);
if (pInfo->pOperator) {
cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable);
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
cleanupAggSup(&pInfo->aggSup);
@ -627,6 +627,42 @@ void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp
}
}
void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo) {
int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
SqlFunctionCtx* pCtx = pSup->pCtx;
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
bool needCleanup = false;
for (int32_t j = 0; j < numOfExprs; ++j) {
needCleanup |= pCtx[j].needCleanup;
}
if (!needCleanup) {
return;
}
for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
SResultRow* pRow = NULL;
SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
SFilePage* page = getBufPage(pBuf, pPos->pos.pageId);
if (page == NULL) {
qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
continue;
}
pRow = (SResultRow*)((char*)page + pPos->pos.offset);
for (int32_t j = 0; j < numOfExprs; ++j) {
pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
if (pCtx[j].fpSet.cleanup) {
pCtx[j].fpSet.cleanup(&pCtx[j]);
}
}
releaseBufPage(pBuf, page);
}
}
void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap) {
int32_t numOfExprs = pSup->numOfExprs;

609
source/libs/executor/src/anomalywindowoperator.c Normal file
View File

@ -0,0 +1,609 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "tanal.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tjson.h"
#include "ttime.h"
#ifdef USE_ANAL
typedef struct {
SArray* blocks; // SSDataBlock*
SArray* windows; // STimeWindow
uint64_t groupId;
int64_t numOfRows;
int32_t curWinIndex;
STimeWindow curWin;
SResultRow* pResultRow;
} SAnomalyWindowSupp;
typedef struct {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SExprSupp scalarSup;
int32_t tsSlotId;
STimeWindowAggSupp twAggSup;
char algoName[TSDB_ANAL_ALGO_NAME_LEN];
char algoUrl[TSDB_ANAL_ALGO_URL_LEN];
char anomalyOpt[TSDB_ANAL_ALGO_OPTION_LEN];
SAnomalyWindowSupp anomalySup;
SWindowRowsSup anomalyWinRowSup;
SColumn anomalyCol;
SStateKeys anomalyKey;
} SAnomalyWindowOperatorInfo;
static void anomalyDestroyOperatorInfo(void* param);
static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes);
static void anomalyAggregateBlocks(SOperatorInfo* pOperator);
static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pBlock);
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode;
SColumnNode* pColNode = (SColumnNode*)(pAnomalyNode->pAnomalyKey);
if (pInfo == NULL || pOperator == NULL) {
code = terrno;
goto _error;
}
if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) {
qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt);
code = TSDB_CODE_ANAL_ALGO_NOT_FOUND;
goto _error;
}
if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) {
qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName);
code = TSDB_CODE_ANAL_ALGO_NOT_LOAD;
goto _error;
}
pOperator->exprSupp.hasWindowOrGroup = true;
pInfo->tsSlotId = ((SColumnNode*)pAnomalyNode->window.pTspk)->slotId;
strncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt));
if (pAnomalyNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
}
size_t keyBufSize = 0;
int32_t num = 0;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc);
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
initBasicInfo(&pInfo->binfo, pResBlock);
code = blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->binfo.inputTsOrder = pAnomalyNode->window.node.inputTsOrder;
pInfo->binfo.outputTsOrder = pAnomalyNode->window.node.outputTsOrder;
pInfo->anomalyCol = extractColumnFromColumnNode(pColNode);
pInfo->anomalyKey.type = pInfo->anomalyCol.type;
pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes;
pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes);
if (pInfo->anomalyKey.pData == NULL) {
goto _error;
}
int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes;
pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize);
pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*));
pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow));
if (pInfo->anomalySup.windows == NULL || pInfo->anomalySup.blocks == NULL || pInfo->anomalySup.pResultRow == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
setOperatorInfo(pOperator, "AnomalyWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, anomalyAggregateNext, NULL, anomalyDestroyOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
QUERY_CHECK_CODE(code, lino, _error);
*pOptrInfo = pOperator;
qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl,
pInfo->anomalyOpt);
return TSDB_CODE_SUCCESS;
_error:
if (pInfo != NULL) {
anomalyDestroyOperatorInfo(pInfo);
}
destroyOperatorAndDownstreams(pOperator, &downstream, 1);
pTaskInfo->code = code;
qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code);
return code;
}
static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SSDataBlock* pRes = pInfo->binfo.pRes;
int64_t st = taosGetTimestampUs();
int32_t numOfBlocks = taosArrayGetSize(pSupp->blocks);
blockDataCleanup(pRes);
while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
if (pSupp->groupId == 0 || pSupp->groupId == pBlock->info.id.groupId) {
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks++;
qDebug("group:%" PRId64 ", blocks:%d, cache block rows:%" PRId64, pSupp->groupId, numOfBlocks, pBlock->info.rows);
code = anomalyCacheBlock(pInfo, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
} else {
qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks);
anomalyAggregateBlocks(pOperator);
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks = 1;
qDebug("group:%" PRId64 ", new group, cache block rows:%" PRId64, pSupp->groupId, pBlock->info.rows);
code = anomalyCacheBlock(pInfo, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pRes->info.rows > 0) {
(*ppRes) = pRes;
qDebug("group:%" PRId64 ", return to upstream, blocks:%d", pRes->info.id.groupId, numOfBlocks);
return code;
}
}
if (numOfBlocks > 0) {
qDebug("group:%" PRId64 ", read finish, blocks:%d", pInfo->anomalySup.groupId, numOfBlocks);
anomalyAggregateBlocks(pOperator);
}
int64_t cost = taosGetTimestampUs() - st;
qDebug("all groups finished, cost:%" PRId64 "us", cost);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
(*ppRes) = (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
return code;
}
static void anomalyDestroyOperatorInfo(void* param) {
SAnomalyWindowOperatorInfo* pInfo = (SAnomalyWindowOperatorInfo*)param;
if (pInfo == NULL) return;
qDebug("anomaly_window operator is destroyed, algo:%s", pInfo->algoName);
cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup);
cleanupExprSupp(&pInfo->scalarSup);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->anomalySup.blocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(pInfo->anomalySup.blocks, i);
blockDataDestroy(pBlock);
}
taosArrayDestroy(pInfo->anomalySup.blocks);
taosArrayDestroy(pInfo->anomalySup.windows);
taosMemoryFreeClear(pInfo->anomalySup.pResultRow);
taosMemoryFreeClear(pInfo->anomalyKey.pData);
taosMemoryFreeClear(param);
}
static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pSrc) {
SSDataBlock* pDst = NULL;
int32_t code = createOneDataBlock(pSrc, true, &pDst);
if (code != 0) return code;
if (pDst == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (taosArrayPush(pInfo->anomalySup.blocks, &pDst) == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return 0;
}
static int32_t anomalyFindWindow(SAnomalyWindowSupp* pSupp, TSKEY key) {
for (int32_t i = pSupp->curWinIndex; i < taosArrayGetSize(pSupp->windows); ++i) {
STimeWindow* pWindow = taosArrayGet(pSupp->windows, i);
if (key >= pWindow->skey && key < pWindow->ekey) {
pSupp->curWin = *pWindow;
pSupp->curWinIndex = i;
return 0;
}
}
return -1;
}
static int32_t anomalyParseJson(SJson* pJson, SArray* pWindows) {
int32_t code = 0;
int32_t rows = 0;
STimeWindow win = {0};
taosArrayClear(pWindows);
tjsonGetInt32ValueFromDouble(pJson, "rows", rows, code);
if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
if (rows <= 0) return 0;
SJson* res = tjsonGetObjectItem(pJson, "res");
if (res == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t ressize = tjsonGetArraySize(res);
if (ressize != rows) return TSDB_CODE_INVALID_JSON_FORMAT;
for (int32_t i = 0; i < rows; ++i) {
SJson* row = tjsonGetArrayItem(res, i);
if (row == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t colsize = tjsonGetArraySize(row);
if (colsize != 2) return TSDB_CODE_INVALID_JSON_FORMAT;
SJson* start = tjsonGetArrayItem(row, 0);
SJson* end = tjsonGetArrayItem(row, 1);
if (start == NULL || end == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
tjsonGetObjectValueBigInt(start, &win.skey);
tjsonGetObjectValueBigInt(end, &win.ekey);
if (win.skey >= win.ekey) {
win.ekey = win.skey + 1;
}
if (taosArrayPush(pWindows, &win) == NULL) return TSDB_CODE_OUT_OF_BUFFER;
}
int32_t numOfWins = taosArrayGetSize(pWindows);
qDebug("anomaly window recevied, total:%d", numOfWins);
for (int32_t i = 0; i < numOfWins; ++i) {
STimeWindow* pWindow = taosArrayGet(pWindows, i);
qDebug("anomaly win:%d [%" PRId64 ", %" PRId64 ")", i, pWindow->skey, pWindow->ekey);
}
return 0;
}
static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) {
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SJson* pJson = NULL;
SAnalBuf analBuf = {.bufType = ANAL_BUF_TYPE_JSON};
char dataBuf[64] = {0};
int32_t code = 0;
int64_t ts = 0;
// int64_t ts = taosGetTimestampMs();
snprintf(analBuf.fileName, sizeof(analBuf.fileName), "%s/tdengine-anomaly-%" PRId64 "-%" PRId64, tsTempDir, ts,
pSupp->groupId);
code = tsosAnalBufOpen(&analBuf, 2);
if (code != 0) goto _OVER;
const char* prec = TSDB_TIME_PRECISION_MILLI_STR;
if (pInfo->anomalyCol.precision == TSDB_TIME_PRECISION_MICRO) prec = TSDB_TIME_PRECISION_MICRO_STR;
if (pInfo->anomalyCol.precision == TSDB_TIME_PRECISION_NANO) prec = TSDB_TIME_PRECISION_NANO_STR;
code = taosAnalBufWriteOptStr(&analBuf, "algo", pInfo->algoName);
if (code != 0) goto _OVER;
code = taosAnalBufWriteOptStr(&analBuf, "prec", prec);
if (code != 0) goto _OVER;
code = taosAnalBufWriteColMeta(&analBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, "ts");
if (code != 0) goto _OVER;
code = taosAnalBufWriteColMeta(&analBuf, 1, pInfo->anomalyCol.type, "val");
if (code != 0) goto _OVER;
code = taosAnalBufWriteDataBegin(&analBuf);
if (code != 0) goto _OVER;
int32_t numOfBlocks = (int32_t)taosArrayGetSize(pSupp->blocks);
// timestamp
code = taosAnalBufWriteColBegin(&analBuf, 0);
if (code != 0) goto _OVER;
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, i);
if (pBlock == NULL) break;
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
if (pTsCol == NULL) break;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
code = taosAnalBufWriteColData(&analBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, &((TSKEY*)pTsCol->pData)[j]);
if (code != 0) goto _OVER;
}
}
code = taosAnalBufWriteColEnd(&analBuf, 0);
if (code != 0) goto _OVER;
// data
code = taosAnalBufWriteColBegin(&analBuf, 1);
if (code != 0) goto _OVER;
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, i);
if (pBlock == NULL) break;
SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pInfo->anomalyCol.slotId);
if (pValCol == NULL) break;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
code = taosAnalBufWriteColData(&analBuf, 1, pValCol->info.type, colDataGetData(pValCol, j));
if (code != 0) goto _OVER;
if (code != 0) goto _OVER;
}
}
code = taosAnalBufWriteColEnd(&analBuf, 1);
if (code != 0) goto _OVER;
code = taosAnalBufWriteDataEnd(&analBuf);
if (code != 0) goto _OVER;
code = taosAnalBufWriteOptStr(&analBuf, "option", pInfo->anomalyOpt);
if (code != 0) goto _OVER;
code = taosAnalBufClose(&analBuf);
if (code != 0) goto _OVER;
pJson = taosAnalSendReqRetJson(pInfo->algoUrl, ANAL_HTTP_TYPE_POST, &analBuf);
if (pJson == NULL) {
code = terrno;
goto _OVER;
}
code = anomalyParseJson(pJson, pSupp->windows);
if (code != 0) goto _OVER;
_OVER:
if (code != 0) {
qError("failed to analysis window since %s", tstrerror(code));
}
taosAnalBufDestroy(&analBuf);
if (pJson != NULL) tjsonDelete(pJson);
return code;
}
static void anomalyAggregateRows(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SWindowRowsSup* pRowSup = &pInfo->anomalyWinRowSup;
SResultRow* pResRow = pSupp->pResultRow;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
if (setResultRowInitCtx(pResRow, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset) == 0) {
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pSupp->curWin, 0);
applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
}
}
static void anomalyBuildResult(SOperatorInfo* pOperator) {
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SSDataBlock* pRes = pInfo->binfo.pRes;
SResultRow* pResRow = pInfo->anomalySup.pResultRow;
doUpdateNumOfRows(pExprSup->pCtx, pResRow, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
copyResultrowToDataBlock(pExprSup->pExprInfo, pExprSup->numOfExprs, pResRow, pExprSup->pCtx, pRes,
pExprSup->rowEntryInfoOffset, pTaskInfo);
pRes->info.rows += pResRow->numOfRows;
clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
}
static void anomalyAggregateBlocks(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnomalyWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SSDataBlock* pRes = pInfo->binfo.pRes;
SAnomalyWindowSupp* pSupp = &pInfo->anomalySup;
SWindowRowsSup* pRowSup = &pInfo->anomalyWinRowSup;
SResultRow* pResRow = pSupp->pResultRow;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
int32_t rowsInWin = 0;
int32_t rowsInBlock = 0;
const int64_t gid = pSupp->groupId;
const int32_t order = pInfo->binfo.inputTsOrder;
int32_t numOfBlocks = (int32_t)taosArrayGetSize(pSupp->blocks);
if (numOfBlocks == 0) goto _OVER;
qDebug("group:%" PRId64 ", aggregate blocks, blocks:%d", pSupp->groupId, numOfBlocks);
pRes->info.id.groupId = pSupp->groupId;
code = anomalyAnalysisWindow(pOperator);
QUERY_CHECK_CODE(code, lino, _OVER);
int32_t numOfWins = taosArrayGetSize(pSupp->windows);
qDebug("group:%" PRId64 ", wins:%d, rows:%" PRId64, pSupp->groupId, numOfWins, pSupp->numOfRows);
for (int32_t w = 0; w < numOfWins; ++w) {
STimeWindow* pWindow = taosArrayGet(pSupp->windows, w);
if (w == 0) {
pSupp->curWin = *pWindow;
pRowSup->win.skey = pSupp->curWin.skey;
}
qDebug("group:%" PRId64 ", win:%d [%" PRId64 ", %" PRId64 ")", pSupp->groupId, w, pWindow->skey, pWindow->ekey);
}
if (numOfWins <= 0) goto _OVER;
if (numOfWins > pRes->info.capacity) {
code = blockDataEnsureCapacity(pRes, numOfWins);
QUERY_CHECK_CODE(code, lino, _OVER);
}
for (int32_t b = 0; b < numOfBlocks; ++b) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, b);
if (pBlock == NULL) break;
pRes->info.scanFlag = pBlock->info.scanFlag;
code = setInputDataBlock(pExprSup, pBlock, order, MAIN_SCAN, true);
if (code != 0) break;
code = blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
if (code != 0) break;
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
if (code != 0) break;
}
SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pInfo->anomalyCol.slotId);
if (pValCol == NULL) break;
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
if (pTsCol == NULL) break;
TSKEY* tsList = (TSKEY*)pTsCol->pData;
bool lastBlock = (b == numOfBlocks - 1);
qTrace("group:%" PRId64 ", block:%d win:%d, riwin:%d riblock:%d, rows:%" PRId64, pSupp->groupId, b,
pSupp->curWinIndex, rowsInWin, rowsInBlock, pBlock->info.rows);
for (int32_t r = 0; r < pBlock->info.rows; ++r) {
TSKEY key = tsList[r];
bool keyInWin = (key >= pSupp->curWin.skey && key < pSupp->curWin.ekey);
bool lastRow = (r == pBlock->info.rows - 1);
if (keyInWin) {
if (r < 5) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d", pSupp->groupId, b,
pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
}
if (rowsInBlock == 0) {
doKeepNewWindowStartInfo(pRowSup, tsList, r, gid);
}
doKeepTuple(pRowSup, tsList[r], gid);
rowsInBlock++;
rowsInWin++;
} else {
if (rowsInBlock > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, agg", pSupp->groupId,
b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
anomalyAggregateRows(pOperator, pBlock);
rowsInBlock = 0;
}
if (rowsInWin > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, build result",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
anomalyBuildResult(pOperator);
rowsInWin = 0;
}
if (anomalyFindWindow(pSupp, tsList[r]) == 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, new window detect",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
doKeepNewWindowStartInfo(pRowSup, tsList, r, gid);
doKeepTuple(pRowSup, tsList[r], gid);
rowsInBlock = 1;
rowsInWin = 1;
} else {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, window not found",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
rowsInBlock = 0;
rowsInWin = 0;
}
}
if (lastRow && rowsInBlock > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, row:%d ts:%" PRId64 ", riwin:%d riblock:%d, agg since lastrow",
pSupp->groupId, b, pSupp->curWinIndex, r, key, rowsInWin, rowsInBlock);
anomalyAggregateRows(pOperator, pBlock);
rowsInBlock = 0;
}
}
if (lastBlock && rowsInWin > 0) {
qTrace("group:%" PRId64 ", block:%d win:%d, riwin:%d riblock:%d, build result since lastblock", pSupp->groupId, b,
pSupp->curWinIndex, rowsInWin, rowsInBlock);
anomalyBuildResult(pOperator);
rowsInWin = 0;
}
}
code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
QUERY_CHECK_CODE(code, lino, _OVER);
_OVER:
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pBlock = taosArrayGetP(pSupp->blocks, i);
qDebug("%s, clear block, pBlock:%p pBlock->pDataBlock:%p", __func__, pBlock, pBlock->pDataBlock);
blockDataDestroy(pBlock);
}
taosArrayClear(pSupp->blocks);
taosArrayClear(pSupp->windows);
pSupp->numOfRows = 0;
pSupp->curWin.ekey = 0;
pSupp->curWin.skey = 0;
pSupp->curWinIndex = 0;
}
#else
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
void destroyForecastInfo(void* param) {}
#endif

16
source/libs/executor/src/eventwindowoperator.c
View File

@ -44,22 +44,6 @@ static int32_t eventWindowAggregateNext(SOperatorInfo* pOperator, SSDataBlock**
static void destroyEWindowOperatorInfo(void* param);
static int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo, SSDataBlock* pBlock);
// todo : move to util
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
uint64_t groupId) {
pRowSup->startRowIndex = rowIndex;
pRowSup->numOfRows = 0;
pRowSup->win.skey = tsList[rowIndex];
pRowSup->groupId = groupId;
}
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
pRowSup->win.ekey = ts;
pRowSup->prevTs = ts;
pRowSup->numOfRows += 1;
pRowSup->groupId = groupId;
}
int32_t createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode,
SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);

8
source/libs/executor/src/executil.c
View File

@ -1795,8 +1795,12 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
SFunctionNode* pFuncNode = (SFunctionNode*)pNode;
SDataType* pType = &pFuncNode->node.resType;
pExp->base.resSchema =
createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName);
const char* pName = pFuncNode->node.aliasName;
if (pFuncNode->funcType == FUNCTION_TYPE_FORECAST_LOW || pFuncNode->funcType == FUNCTION_TYPE_FORECAST_HIGH ||
pFuncNode->funcType == FUNCTION_TYPE_FORECAST_ROWTS) {
pName = pFuncNode->functionName;
}
pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pName);
tExprNode* pExprNode = pExp->pExpr;

663
source/libs/executor/src/forecastoperator.c Normal file
View File

@ -0,0 +1,663 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "storageapi.h"
#include "tanal.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tfill.h"
#include "ttime.h"
#ifdef USE_ANAL
typedef struct {
char algoName[TSDB_ANAL_ALGO_NAME_LEN];
char algoUrl[TSDB_ANAL_ALGO_URL_LEN];
char algoOpt[TSDB_ANAL_ALGO_OPTION_LEN];
int64_t maxTs;
int64_t minTs;
int64_t numOfRows;
uint64_t groupId;
int32_t numOfBlocks;
int32_t optRows;
int16_t resTsSlot;
int16_t resValSlot;
int16_t resLowSlot;
int16_t resHighSlot;
int16_t inputTsSlot;
int16_t inputValSlot;
int8_t inputValType;
int8_t inputPrecision;
SAnalBuf analBuf;
} SForecastSupp;
typedef struct SForecastOperatorInfo {
SSDataBlock* pRes;
SExprSupp scalarSup; // scalar calculation
SForecastSupp forecastSupp;
} SForecastOperatorInfo;
static void destroyForecastInfo(void* param);
static FORCE_INLINE int32_t forecastEnsureBlockCapacity(SSDataBlock* pBlock, int32_t newRowsNum) {
if (pBlock->info.rows < pBlock->info.capacity) {
return TSDB_CODE_SUCCESS;
}
int32_t code = blockDataEnsureCapacity(pBlock, newRowsNum);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
return TSDB_CODE_SUCCESS;
}
static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnalBuf* pBuf = &pSupp->analBuf;
qDebug("block:%d, %p rows:%" PRId64, pSupp->numOfBlocks, pBlock, pBlock->info.rows);
pSupp->numOfBlocks++;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pSupp->inputValSlot);
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSupp->inputTsSlot);
if (pTsCol == NULL || pValCol == NULL) break;
int64_t ts = ((TSKEY*)pTsCol->pData)[j];
char* val = colDataGetData(pValCol, j);
int16_t valType = pValCol->info.type;
pSupp->minTs = MIN(pSupp->minTs, ts);
pSupp->maxTs = MAX(pSupp->maxTs, ts);
pSupp->numOfRows++;
code = taosAnalBufWriteColData(pBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, &ts);
if (TSDB_CODE_SUCCESS != code) return code;
code = taosAnalBufWriteColData(pBuf, 1, valType, val);
if (TSDB_CODE_SUCCESS != code) return code;
}
return 0;
}
static int32_t forecastCloseBuf(SForecastSupp* pSupp) {
SAnalBuf* pBuf = &pSupp->analBuf;
int32_t code = 0;
for (int32_t i = 0; i < 2; ++i) {
code = taosAnalBufWriteColEnd(pBuf, i);
if (code != 0) return code;
}
code = taosAnalBufWriteDataEnd(pBuf);
if (code != 0) return code;
int32_t len = strlen(pSupp->algoOpt);
int64_t every = (pSupp->maxTs - pSupp->minTs) / (pSupp->numOfRows + 1);
int64_t start = pSupp->maxTs + every;
bool hasStart = taosAnalGetOptStr(pSupp->algoOpt, "start", NULL, 0);
if (!hasStart) {
qDebug("forecast start not found from %s, use %" PRId64, pSupp->algoOpt, start);
code = taosAnalBufWriteOptInt(pBuf, "start", start);
if (code != 0) return code;
}
bool hasEvery = taosAnalGetOptStr(pSupp->algoOpt, "every", NULL, 0);
if (!hasEvery) {
qDebug("forecast every not found from %s, use %" PRId64, pSupp->algoOpt, every);
code = taosAnalBufWriteOptInt(pBuf, "every", every);
if (code != 0) return code;
}
code = taosAnalBufWriteOptStr(pBuf, "option", pSupp->algoOpt);
if (code != 0) return code;
code = taosAnalBufClose(pBuf);
return code;
}
static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock) {
SAnalBuf* pBuf = &pSupp->analBuf;
int32_t resCurRow = pBlock->info.rows;
int8_t tmpI8;
int16_t tmpI16;
int32_t tmpI32;
int64_t tmpI64;
float tmpFloat;
double tmpDouble;
int32_t code = 0;
SColumnInfoData* pResValCol = taosArrayGet(pBlock->pDataBlock, pSupp->resValSlot);
if (NULL == pResValCol) return TSDB_CODE_OUT_OF_RANGE;
SColumnInfoData* pResTsCol = (pSupp->resTsSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL);
SColumnInfoData* pResLowCol = (pSupp->resLowSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL);
SColumnInfoData* pResHighCol =
(pSupp->resHighSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resHighSlot) : NULL);
SJson* pJson = taosAnalSendReqRetJson(pSupp->algoUrl, ANAL_HTTP_TYPE_POST, pBuf);
if (pJson == NULL) return terrno;
int32_t rows = 0;
tjsonGetInt32ValueFromDouble(pJson, "rows", rows, code);
if (code < 0) goto _OVER;
if (rows <= 0) goto _OVER;
SJson* res = tjsonGetObjectItem(pJson, "res");
if (res == NULL) goto _OVER;
int32_t ressize = tjsonGetArraySize(res);
bool returnConf = (pSupp->resHighSlot != -1 || pSupp->resLowSlot != -1);
if (returnConf) {
if (ressize != 4) goto _OVER;
} else if (ressize != 2) {
goto _OVER;
}
if (pResTsCol != NULL) {
resCurRow = pBlock->info.rows;
SJson* tsJsonArray = tjsonGetArrayItem(res, 0);
if (tsJsonArray == NULL) goto _OVER;
int32_t tsSize = tjsonGetArraySize(tsJsonArray);
if (tsSize != rows) goto _OVER;
for (int32_t i = 0; i < tsSize; ++i) {
SJson* tsJson = tjsonGetArrayItem(tsJsonArray, i);
tjsonGetObjectValueBigInt(tsJson, &tmpI64);
colDataSetInt64(pResTsCol, resCurRow, &tmpI64);
resCurRow++;
}
}
if (pResLowCol != NULL) {
resCurRow = pBlock->info.rows;
SJson* lowJsonArray = tjsonGetArrayItem(res, 2);
if (lowJsonArray == NULL) goto _OVER;
int32_t lowSize = tjsonGetArraySize(lowJsonArray);
if (lowSize != rows) goto _OVER;
for (int32_t i = 0; i < lowSize; ++i) {
SJson* lowJson = tjsonGetArrayItem(lowJsonArray, i);
tjsonGetObjectValueDouble(lowJson, &tmpDouble);
tmpFloat = (float)tmpDouble;
colDataSetFloat(pResLowCol, resCurRow, &tmpFloat);
resCurRow++;
}
}
if (pResHighCol != NULL) {
resCurRow = pBlock->info.rows;
SJson* highJsonArray = tjsonGetArrayItem(res, 3);
if (highJsonArray == NULL) goto _OVER;
int32_t highSize = tjsonGetArraySize(highJsonArray);
if (highSize != rows) goto _OVER;
for (int32_t i = 0; i < highSize; ++i) {
SJson* highJson = tjsonGetArrayItem(highJsonArray, i);
tjsonGetObjectValueDouble(highJson, &tmpDouble);
tmpFloat = (float)tmpDouble;
colDataSetFloat(pResHighCol, resCurRow, &tmpFloat);
resCurRow++;
}
}
resCurRow = pBlock->info.rows;
SJson* valJsonArray = tjsonGetArrayItem(res, 1);
if (valJsonArray == NULL) goto _OVER;
int32_t valSize = tjsonGetArraySize(valJsonArray);
if (valSize != rows) goto _OVER;
for (int32_t i = 0; i < valSize; ++i) {
SJson* valJson = tjsonGetArrayItem(valJsonArray, i);
tjsonGetObjectValueDouble(valJson, &tmpDouble);
switch (pSupp->inputValType) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
tmpI8 = (int8_t)tmpDouble;
colDataSetInt8(pResValCol, resCurRow, &tmpI8);
break;
}
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
tmpI16 = (int16_t)tmpDouble;
colDataSetInt16(pResValCol, resCurRow, &tmpI16);
break;
}
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: {
tmpI32 = (int32_t)tmpDouble;
colDataSetInt32(pResValCol, resCurRow, &tmpI32);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_BIGINT: {
tmpI64 = (int64_t)tmpDouble;
colDataSetInt64(pResValCol, resCurRow, &tmpI64);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
tmpFloat = (float)tmpDouble;
colDataSetFloat(pResValCol, resCurRow, &tmpFloat);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
colDataSetDouble(pResValCol, resCurRow, &tmpDouble);
break;
}
default:
code = TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
goto _OVER;
}
resCurRow++;
}
// for (int32_t i = rows; i < pSupp->optRows; ++i) {
// colDataSetNNULL(pResValCol, rows, (pSupp->optRows - rows));
// if (pResTsCol != NULL) {
// colDataSetNNULL(pResTsCol, rows, (pSupp->optRows - rows));
// }
// if (pResLowCol != NULL) {
// colDataSetNNULL(pResLowCol, rows, (pSupp->optRows - rows));
// }
// if (pResHighCol != NULL) {
// colDataSetNNULL(pResHighCol, rows, (pSupp->optRows - rows));
// }
// }
// if (rows == pSupp->optRows) {
// pResValCol->hasNull = false;
// }
pBlock->info.rows += rows;
if (pJson != NULL) tjsonDelete(pJson);
return 0;
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
if (code == 0) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
}
qError("failed to perform forecast finalize since %s", tstrerror(code));
return TSDB_CODE_INVALID_JSON_FORMAT;
}
static int32_t forecastAggregateBlocks(SForecastSupp* pSupp, SSDataBlock* pResBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SAnalBuf* pBuf = &pSupp->analBuf;
code = forecastCloseBuf(pSupp);
QUERY_CHECK_CODE(code, lino, _end);
code = forecastEnsureBlockCapacity(pResBlock, 1);
QUERY_CHECK_CODE(code, lino, _end);
code = forecastAnalysis(pSupp, pResBlock);
QUERY_CHECK_CODE(code, lino, _end);
uInfo("block:%d, forecast finalize", pSupp->numOfBlocks);
_end:
pSupp->numOfBlocks = 0;
taosAnalBufDestroy(&pSupp->analBuf);
return code;
}
static int32_t forecastNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SForecastOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pResBlock = pInfo->pRes;
SForecastSupp* pSupp = &pInfo->forecastSupp;
SAnalBuf* pBuf = &pSupp->analBuf;
int64_t st = taosGetTimestampUs();
int32_t numOfBlocks = pSupp->numOfBlocks;
blockDataCleanup(pResBlock);
while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
if (pSupp->groupId == 0 || pSupp->groupId == pBlock->info.id.groupId) {
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks++;
qDebug("group:%" PRId64 ", blocks:%d, cache block rows:%" PRId64, pSupp->groupId, numOfBlocks, pBlock->info.rows);
code = forecastCacheBlock(pSupp, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
} else {
qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks);
forecastAggregateBlocks(pSupp, pResBlock);
pSupp->groupId = pBlock->info.id.groupId;
numOfBlocks = 1;
qDebug("group:%" PRId64 ", new group, cache block rows:%" PRId64, pSupp->groupId, pBlock->info.rows);
code = forecastCacheBlock(pSupp, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
}
if (pResBlock->info.rows > 0) {
(*ppRes) = pResBlock;
qDebug("group:%" PRId64 ", return to upstream, blocks:%d", pResBlock->info.id.groupId, numOfBlocks);
return code;
}
}
if (numOfBlocks > 0) {
qDebug("group:%" PRId64 ", read finish, blocks:%d", pSupp->groupId, numOfBlocks);
forecastAggregateBlocks(pSupp, pResBlock);
}
int64_t cost = taosGetTimestampUs() - st;
qDebug("all groups finished, cost:%" PRId64 "us", cost);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
(*ppRes) = (pResBlock->info.rows == 0) ? NULL : pResBlock;
return code;
}
static int32_t forecastParseOutput(SForecastSupp* pSupp, SExprSupp* pExprSup) {
pSupp->resLowSlot = -1;
pSupp->resHighSlot = -1;
pSupp->resTsSlot = -1;
pSupp->resValSlot = -1;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST) {
pSupp->resValSlot = dstSlot;
} else if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST_ROWTS) {
pSupp->resTsSlot = dstSlot;
} else if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST_LOW) {
pSupp->resLowSlot = dstSlot;
} else if (pExprInfo->pExpr->_function.functionType == FUNCTION_TYPE_FORECAST_HIGH) {
pSupp->resHighSlot = dstSlot;
} else {
}
}
return 0;
}
static int32_t forecastParseInput(SForecastSupp* pSupp, SNodeList* pFuncs) {
SNode* pNode = NULL;
pSupp->inputTsSlot = -1;
pSupp->inputValSlot = -1;
pSupp->inputValType = -1;
pSupp->inputPrecision = -1;
FOREACH(pNode, pFuncs) {
if ((nodeType(pNode) == QUERY_NODE_TARGET) && (nodeType(((STargetNode*)pNode)->pExpr) == QUERY_NODE_FUNCTION)) {
SFunctionNode* pFunc = (SFunctionNode*)((STargetNode*)pNode)->pExpr;
int32_t numOfParam = LIST_LENGTH(pFunc->pParameterList);
if (pFunc->funcType == FUNCTION_TYPE_FORECAST) {
if (numOfParam == 3) {
SNode* p1 = nodesListGetNode(pFunc->pParameterList, 0);
SNode* p2 = nodesListGetNode(pFunc->pParameterList, 1);
SNode* p3 = nodesListGetNode(pFunc->pParameterList, 2);
if (p1 == NULL || p2 == NULL || p3 == NULL) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p1->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p2->type != QUERY_NODE_VALUE) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p3->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
SColumnNode* pValNode = (SColumnNode*)p1;
SValueNode* pOptNode = (SValueNode*)p2;
SColumnNode* pTsNode = (SColumnNode*)p3;
pSupp->inputTsSlot = pTsNode->slotId;
pSupp->inputPrecision = pTsNode->node.resType.precision;
pSupp->inputValSlot = pValNode->slotId;
pSupp->inputValType = pValNode->node.resType.type;
tstrncpy(pSupp->algoOpt, pOptNode->literal, sizeof(pSupp->algoOpt));
} else if (numOfParam == 2) {
SNode* p1 = nodesListGetNode(pFunc->pParameterList, 0);
SNode* p2 = nodesListGetNode(pFunc->pParameterList, 1);
if (p1 == NULL || p2 == NULL) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p1->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
if (p2->type != QUERY_NODE_COLUMN) return TSDB_CODE_PLAN_INTERNAL_ERROR;
SColumnNode* pValNode = (SColumnNode*)p1;
SColumnNode* pTsNode = (SColumnNode*)p2;
pSupp->inputTsSlot = pTsNode->slotId;
pSupp->inputPrecision = pTsNode->node.resType.precision;
pSupp->inputValSlot = pValNode->slotId;
pSupp->inputValType = pValNode->node.resType.type;
tstrncpy(pSupp->algoOpt, "algo=arima", TSDB_ANAL_ALGO_OPTION_LEN);
} else {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
}
}
}
return 0;
}
static int32_t forecastParseAlgo(SForecastSupp* pSupp) {
pSupp->maxTs = 0;
pSupp->minTs = INT64_MAX;
pSupp->numOfRows = 0;
if (!taosAnalGetOptStr(pSupp->algoOpt, "algo", pSupp->algoName, sizeof(pSupp->algoName))) {
qError("failed to get forecast algorithm name from %s", pSupp->algoOpt);
return TSDB_CODE_ANAL_ALGO_NOT_FOUND;
}
if (taosAnalGetAlgoUrl(pSupp->algoName, ANAL_ALGO_TYPE_FORECAST, pSupp->algoUrl, sizeof(pSupp->algoUrl)) != 0) {
qError("failed to get forecast algorithm url from %s", pSupp->algoName);
return TSDB_CODE_ANAL_ALGO_NOT_LOAD;
}
return 0;
}
static int32_t forecastCreateBuf(SForecastSupp* pSupp) {
SAnalBuf* pBuf = &pSupp->analBuf;
int64_t ts = 0; // taosGetTimestampMs();
pBuf->bufType = ANAL_BUF_TYPE_JSON_COL;
snprintf(pBuf->fileName, sizeof(pBuf->fileName), "%s/tdengine-forecast-%" PRId64, tsTempDir, ts);
int32_t code = tsosAnalBufOpen(pBuf, 2);
if (code != 0) goto _OVER;
code = taosAnalBufWriteOptStr(pBuf, "algo", pSupp->algoName);
if (code != 0) goto _OVER;
bool returnConf = (pSupp->resHighSlot == -1 || pSupp->resLowSlot == -1);
code = taosAnalBufWriteOptStr(pBuf, "return_conf", returnConf ? "true" : "false");
if (code != 0) goto _OVER;
bool hasAlpha = taosAnalGetOptStr(pSupp->algoOpt, "alpha", NULL, 0);
if (!hasAlpha) {
qDebug("forecast alpha not found from %s, use default:%f", pSupp->algoOpt, ANAL_FORECAST_DEFAULT_ALPHA);
code = taosAnalBufWriteOptFloat(pBuf, "alpha", ANAL_FORECAST_DEFAULT_ALPHA);
if (code != 0) goto _OVER;
}
char tmpOpt[32] = {0};
bool hasParam = taosAnalGetOptStr(pSupp->algoOpt, "param", tmpOpt, sizeof(tmpOpt));
if (!hasParam) {
qDebug("forecast param not found from %s, use default:%s", pSupp->algoOpt, ANAL_FORECAST_DEFAULT_PARAM);
code = taosAnalBufWriteOptStr(pBuf, "param", ANAL_FORECAST_DEFAULT_PARAM);
if (code != 0) goto _OVER;
}
bool hasPeriod = taosAnalGetOptInt(pSupp->algoOpt, "period", NULL);
if (!hasPeriod) {
qDebug("forecast period not found from %s, use default:%d", pSupp->algoOpt, ANAL_FORECAST_DEFAULT_PERIOD);
code = taosAnalBufWriteOptInt(pBuf, "period", ANAL_FORECAST_DEFAULT_PERIOD);
if (code != 0) goto _OVER;
}
bool hasRows = taosAnalGetOptInt(pSupp->algoOpt, "rows", &pSupp->optRows);
if (!hasRows) {
pSupp->optRows = ANAL_FORECAST_DEFAULT_ROWS;
qDebug("forecast rows not found from %s, use default:%d", pSupp->algoOpt, pSupp->optRows);
code = taosAnalBufWriteOptInt(pBuf, "forecast_rows", pSupp->optRows);
if (code != 0) goto _OVER;
}
const char* prec = TSDB_TIME_PRECISION_MILLI_STR;
if (pSupp->inputPrecision == TSDB_TIME_PRECISION_MICRO) prec = TSDB_TIME_PRECISION_MICRO_STR;
if (pSupp->inputPrecision == TSDB_TIME_PRECISION_NANO) prec = TSDB_TIME_PRECISION_NANO_STR;
code = taosAnalBufWriteOptStr(pBuf, "prec", prec);
if (code != 0) goto _OVER;
if (returnConf) {
bool hasConf = taosAnalGetOptStr(pSupp->algoOpt, "conf", NULL, 0);
if (!hasConf) {
qDebug("forecast conf not found from %s, use default:%d", pSupp->algoOpt, ANAL_FORECAST_DEFAULT_CONF);
code = taosAnalBufWriteOptInt(pBuf, "conf", ANAL_FORECAST_DEFAULT_CONF);
if (code != 0) goto _OVER;
}
}
code = taosAnalBufWriteColMeta(pBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, "ts");
if (code != 0) goto _OVER;
code = taosAnalBufWriteColMeta(pBuf, 1, pSupp->inputValType, "val");
if (code != 0) goto _OVER;
code = taosAnalBufWriteDataBegin(pBuf);
if (code != 0) goto _OVER;
for (int32_t i = 0; i < 2; ++i) {
code = taosAnalBufWriteColBegin(pBuf, i);
if (code != 0) goto _OVER;
}
_OVER:
if (code != 0) {
taosAnalBufClose(pBuf);
taosAnalBufDestroy(pBuf);
}
return code;
}
int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = 0;
int32_t lino = 0;
SForecastOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SForecastOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL || pInfo == NULL) {
code = terrno;
goto _error;
}
SForecastSupp* pSupp = &pInfo->forecastSupp;
SForecastFuncPhysiNode* pForecastPhyNode = (SForecastFuncPhysiNode*)pPhyNode;
SExprSupp* pExprSup = &pOperator->exprSupp;
int32_t numOfExprs = 0;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pForecastPhyNode->pFuncs, NULL, &pExprInfo, &numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(pExprSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
if (pForecastPhyNode->pExprs != NULL) {
int32_t num = 0;
SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pForecastPhyNode->pExprs, NULL, &pScalarExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
}
code = filterInitFromNode((SNode*)pForecastPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastParseInput(pSupp, pForecastPhyNode->pFuncs);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastParseOutput(pSupp, pExprSup);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastParseAlgo(pSupp);
QUERY_CHECK_CODE(code, lino, _error);
code = forecastCreateBuf(pSupp);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pRes = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno);
setOperatorInfo(pOperator, "ForecastOperator", QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, forecastNext, NULL, destroyForecastInfo, optrDefaultBufFn,
NULL, optrDefaultGetNextExtFn, NULL);
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
QUERY_CHECK_CODE(code, lino, _error);
*pOptrInfo = pOperator;
qDebug("forecast env is initialized, option:%s", pSupp->algoOpt);
return TSDB_CODE_SUCCESS;
_error:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (pInfo != NULL) destroyForecastInfo(pInfo);
destroyOperatorAndDownstreams(pOperator, &downstream, 1);
pTaskInfo->code = code;
return code;
}
static void destroyForecastInfo(void* param) {
SForecastOperatorInfo* pInfo = (SForecastOperatorInfo*)param;
blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
cleanupExprSupp(&pInfo->scalarSup);
taosAnalBufDestroy(&pInfo->forecastSupp.analBuf);
taosMemoryFreeClear(param);
}
#else
int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
#endif

2
source/libs/executor/src/groupcacheoperator.c
View File

@ -522,7 +522,7 @@ static int32_t buildGroupCacheBaseBlock(SSDataBlock** ppDst, SSDataBlock* pSrc)
(*ppDst)->pDataBlock = taosArrayDup(pSrc->pDataBlock, NULL);
if (NULL == (*ppDst)->pDataBlock) {
taosMemoryFree(*ppDst);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
TAOS_MEMCPY(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info));
blockDataDeepClear(*ppDst);

4
source/libs/executor/src/operator.c
View File

@ -619,6 +619,8 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
code = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) {
code = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC == type) {
code = createForecastOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) {
code = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE == type) {
@ -629,6 +631,8 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
code = createStreamCountAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT == type) {
code = createCountwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY == type) {
code = createAnomalywindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else {
code = TSDB_CODE_INVALID_PARA;
pTaskInfo->code = code;

2
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -49,7 +49,7 @@
#define STREAM_SESSION_OP_CHECKPOINT_NAME "StreamSessionOperator_Checkpoint"
#define STREAM_STATE_OP_CHECKPOINT_NAME "StreamStateOperator_Checkpoint"
#define MAX_STREAM_HISTORY_RESULT 100000000
#define MAX_STREAM_HISTORY_RESULT 20000000
typedef struct SStateWindowInfo {
SResultWindowInfo winInfo;

25
source/libs/executor/src/timewindowoperator.c
View File

@ -89,14 +89,14 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo
return setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
}
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
pRowSup->win.ekey = ts;
pRowSup->prevTs = ts;
pRowSup->numOfRows += 1;
pRowSup->groupId = groupId;
}
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
uint64_t groupId) {
pRowSup->startRowIndex = rowIndex;
pRowSup->numOfRows = 0;
@ -1229,10 +1229,9 @@ static void destroyStateWindowOperatorInfo(void* param) {
SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo);
taosMemoryFreeClear(pInfo->stateKey.pData);
if (pInfo->pOperator != NULL) {
cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable);
if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
@ -1257,10 +1256,9 @@ void destroyIntervalOperatorInfo(void* param) {
SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo);
if (pInfo->pOperator != NULL) {
cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable);
if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
@ -1764,10 +1762,9 @@ void destroySWindowOperatorInfo(void* param) {
cleanupBasicInfo(&pInfo->binfo);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
if (pInfo->pOperator != NULL) {
cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable);
if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf,
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}

2
source/libs/function/inc/builtinsimpl.h
View File

@ -138,6 +138,8 @@ int32_t diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t diffFunction(SqlFunctionCtx* pCtx);
int32_t diffFunctionByRow(SArray* pCtx);
bool getForecastConfEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t derivativeFunction(SqlFunctionCtx* pCtx);

1
source/libs/function/inc/functionMgtInt.h
View File

@ -58,6 +58,7 @@ extern "C" {
#define FUNC_MGT_TSMA_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(29)
#define FUNC_MGT_COUNT_LIKE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found
#define FUNC_MGT_PROCESS_BY_ROW FUNC_MGT_FUNC_CLASSIFICATION_MASK(31)
#define FUNC_MGT_FORECAST_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(32)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

46
source/libs/function/src/builtins.c
View File

@ -16,9 +16,10 @@
#include "builtins.h"
#include "builtinsimpl.h"
#include "cJSON.h"
#include "geomFunc.h"
#include "querynodes.h"
#include "scalar.h"
#include "geomFunc.h"
#include "tanal.h"
#include "taoserror.h"
#include "ttime.h"
@ -237,7 +238,7 @@ static int32_t addTimezoneParam(SNodeList* pList) {
return terrno;
}
varDataSetLen(pVal->datum.p, len);
(void)strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
tstrncpy(varDataVal(pVal->datum.p), pVal->literal, len + 1);
code = nodesListAppend(pList, (SNode*)pVal);
if (TSDB_CODE_SUCCESS != code) {
@ -2078,6 +2079,47 @@ static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateUniqueMode(pFunc, pErrBuf, len, false);
}
static int32_t translateForecast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (2 != numOfParams && 1 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, "FORECAST require 1 or 2 parameters");
}
uint8_t valType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_MATHABLE_TYPE(valType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, "FORECAST only support mathable column");
}
if (numOfParams == 2) {
uint8_t optionType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 1))->type;
if (TSDB_DATA_TYPE_BINARY != optionType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, "FORECAST option should be varchar");
}
SNode* pOption = nodesListGetNode(pFunc->pParameterList, 1);
if (QUERY_NODE_VALUE != nodeType(pOption)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, "FORECAST option should be value");
}
SValueNode* pValue = (SValueNode*)pOption;
if (!taosAnalGetOptStr(pValue->literal, "algo", NULL, 0) != 0) {
return invaildFuncParaValueErrMsg(pErrBuf, len, "FORECAST option should include algo field");
}
pValue->notReserved = true;
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[valType].bytes, .type = valType};
return TSDB_CODE_SUCCESS;
}
static int32_t translateForecastConf(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_FLOAT].bytes, .type = TSDB_DATA_TYPE_FLOAT};
return TSDB_CODE_SUCCESS;
}
static EFuncReturnRows forecastEstReturnRows(SFunctionNode* pFunc) { return FUNC_RETURN_ROWS_N; }
static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (numOfParams > 2) {

46
source/libs/function/src/builtinsimpl.c
View File

@ -18,6 +18,7 @@
#include "function.h"
#include "query.h"
#include "querynodes.h"
#include "tanal.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tdigest.h"
@ -2153,7 +2154,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _fin_error;
}
@ -3578,6 +3579,11 @@ bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool f
}
}
bool getForecastConfEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(float);
return true;
}
int32_t diffResultIsNull(SqlFunctionCtx* pCtx, SFuncInputRow* pRow){
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
@ -3676,7 +3682,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) {
for (int i = 0; i < diffColNum; ++i) {
SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
if (NULL == pCtx) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
funcInputUpdate(pCtx);
@ -3690,7 +3696,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) {
SqlFunctionCtx* pCtx0 = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, 0);
SFuncInputRow* pRow0 = (SFuncInputRow*)taosArrayGet(pRows, 0);
if (NULL == pCtx0 || NULL == pRow0) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
int32_t startOffset = pCtx0->offset;
@ -3708,7 +3714,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) {
SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i);
if (NULL == pCtx || NULL == pRow) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
code = funcInputGetNextRow(pCtx, pRow, &result);
@ -3731,7 +3737,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) {
SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i);
if (NULL == pCtx || NULL == pRow) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
if ((keepNull || hasNotNullValue) && !isFirstRow(pCtx, pRow)){
@ -3753,7 +3759,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) {
for (int i = 0; i < diffColNum; ++i) {
SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
if (NULL == pCtx) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@ -4430,7 +4436,7 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
@ -4620,7 +4626,7 @@ int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
@ -4970,10 +4976,10 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t len;
char buf[512] = {0};
if (!pInfo->normalized) {
len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}",
len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}",
pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count);
} else {
len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower,
len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower,
pInfo->bins[i].upper, pInfo->bins[i].percentage);
}
varDataSetLen(buf, len);
@ -5003,7 +5009,7 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
code = colDataSetVal(pCol, pBlock->info.rows, res, false);
@ -5236,7 +5242,7 @@ int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
code = terrno;
goto _exit;
}
@ -6601,7 +6607,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
compRatio = pData->totalSize * 100 / (double)totalRawSize;
}
int32_t len = sprintf(st + VARSTR_HEADER_SIZE,
int32_t len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
"Total_Blocks=[%d] Total_Size=[%.2f KiB] Average_size=[%.2f KiB] Compression_Ratio=[%.2f %c]",
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%');
@ -6616,7 +6622,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
avgRows = pData->totalRows / pData->numOfBlocks;
}
len = sprintf(st + VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]",
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]",
pData->totalRows, pData->minRows, pData->maxRows, avgRows);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
@ -6624,14 +6630,14 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return code;
}
len = sprintf(st + VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows);
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables,
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables,
pData->numOfFiles, pData->numOfVgroups);
varDataSetLen(st, len);
@ -6640,7 +6646,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return code;
}
len = sprintf(st + VARSTR_HEADER_SIZE,
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
"--------------------------------------------------------------------------------");
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
@ -6667,7 +6673,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t bucketRange = ceil(((double) (pData->defMaxRows - pData->defMinRows)) / numOfBuckets);
for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) {
len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
int32_t num = 0;
if (pData->blockRowsHisto[i] > 0) {
@ -6675,13 +6681,13 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
}
for (int32_t j = 0; j < num; ++j) {
int32_t x = sprintf(st + VARSTR_HEADER_SIZE + len, "%c", '|');
int32_t x = snprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, "%c", '|');
len += x;
}
if (pData->blockRowsHisto[i] > 0) {
double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks;
len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
len += snprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
}
varDataSetLen(st, len);

27
source/libs/function/src/functionMgt.c
View File

@ -232,6 +232,15 @@ bool fmIsInterpFunc(int32_t funcId) {
bool fmIsInterpPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INTERP_PC_FUNC); }
bool fmIsForecastFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_FORECAST == funcMgtBuiltins[funcId].type;
}
bool fmIsForecastPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORECAST_PC_FUNC); }
bool fmIsLastRowFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
@ -408,7 +417,7 @@ static int32_t createColumnByFunc(const SFunctionNode* pFunc, SColumnNode** ppCo
if (NULL == *ppCol) {
return code;
}
(void)strcpy((*ppCol)->colName, pFunc->node.aliasName);
tstrncpy((*ppCol)->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN);
(*ppCol)->node.resType = pFunc->node.resType;
return TSDB_CODE_SUCCESS;
}
@ -437,11 +446,11 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
(*pPartialFunc)->hasOriginalFunc = true;
(*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0};
int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
int32_t len = snprintf(name, sizeof(name), "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
if (taosHashBinary(name, len) < 0) {
return TSDB_CODE_FAILED;
}
(void)strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1);
tstrncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN);
(*pPartialFunc)->hasPk = pSrcFunc->hasPk;
(*pPartialFunc)->pkBytes = pSrcFunc->pkBytes;
return TSDB_CODE_SUCCESS;
@ -475,7 +484,7 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN
}
}
if (TSDB_CODE_SUCCESS == code) {
(void)strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName);
tstrncpy(pFunc->node.aliasName, pPartialFunc->node.aliasName, TSDB_COL_NAME_LEN);
}
if (TSDB_CODE_SUCCESS == code) {
@ -504,7 +513,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
if (fmIsSameInOutType(pSrcFunc->funcId)) {
pFunc->node.resType = pSrcFunc->node.resType;
}
(void)strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName);
tstrncpy(pFunc->node.aliasName, pSrcFunc->node.aliasName, TSDB_COL_NAME_LEN);
}
if (TSDB_CODE_SUCCESS == code) {
@ -558,8 +567,8 @@ static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pSt
nodesDestroyList(pParams);
return code;
}
(void)strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName);
(void)strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias);
tstrncpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN);
tstrncpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN);
}
return TSDB_CODE_SUCCESS;
}
@ -605,8 +614,8 @@ static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pSta
nodesDestroyList(pParams);
return code;
}
(void)strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName);
(void)strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias);
tstrncpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN);
tstrncpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN);
}
return TSDB_CODE_SUCCESS;
}

8
source/libs/function/src/tscript.c
View File

@ -92,7 +92,7 @@ void taosValueToLuaType(lua_State *lua, int32_t type, char *val) {
int taosLoadScriptInit(void* pInit) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_init", pCtx->funcName);
snprintf(funcName, MAX_FUNC_NAME, "%s_init", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
@ -106,7 +106,7 @@ void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iByt
int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes) {
ScriptCtx* pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_add", pCtx->funcName);
snprintf(funcName, MAX_FUNC_NAME, "%s_add", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
@ -143,7 +143,7 @@ void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iByt
void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutput, int32_t* numOfOutput) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_merge", pCtx->funcName);
snprintf(funcName, MAX_FUNC_NAME, "%s_merge", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
@ -167,7 +167,7 @@ void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutp
void taosLoadScriptFinalize(void *pInit,int64_t key, char *pOutput, int32_t* numOfOutput) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_finalize", pCtx->funcName);
snprintf(funcName, MAX_FUNC_NAME, "%s_finalize", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);

22
source/libs/function/src/tudf.c
View File

@ -143,10 +143,10 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
char udfdPathLdLib[1024] = {0};
size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath);
strncpy(udfdPathLdLib, tsUdfdLdLibPath, tListLen(udfdPathLdLib));
tstrncpy(udfdPathLdLib, tsUdfdLdLibPath, sizeof(udfdPathLdLib) < sizeof(tsUdfdLdLibPath) ? sizeof(udfdPathLdLib) : sizeof(tsUdfdLdLibPath));
udfdPathLdLib[udfdLdLibPathLen] = ':';
strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1);
tstrncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1);
if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) {
fnInfo("[UDFD]udfd LD_LIBRARY_PATH: %s", udfdPathLdLib);
} else {
@ -158,10 +158,12 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
char *taosFqdnEnvItem = NULL;
char *taosFqdn = getenv("TAOS_FQDN");
if (taosFqdn != NULL) {
taosFqdnEnvItem = taosMemoryMalloc(strlen("TAOS_FQDN=") + strlen(taosFqdn) + 1);
int subLen = strlen(taosFqdn);
int len = strlen("TAOS_FQDN=") + subLen + 1;
taosFqdnEnvItem = taosMemoryMalloc(len);
if (taosFqdnEnvItem != NULL) {
strcpy(taosFqdnEnvItem, "TAOS_FQDN=");
TAOS_STRCAT(taosFqdnEnvItem, taosFqdn);
tstrncpy(taosFqdnEnvItem, "TAOS_FQDN=", len);
TAOS_STRNCAT(taosFqdnEnvItem, taosFqdn, subLen);
fnInfo("[UDFD]Succsess to set TAOS_FQDN:%s", taosFqdn);
} else {
fnError("[UDFD]Failed to allocate memory for TAOS_FQDN");
@ -1072,7 +1074,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
int32_t code = 0, line = 0;
uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
tstrncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
int32_t stubIndex = taosArraySearchIdx(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
if (stubIndex != -1) {
SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex);
@ -1105,7 +1107,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
code = doSetupUdf(udfName, pHandle);
if (code == TSDB_CODE_SUCCESS) {
SUdfcFuncStub stub = {0};
strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN);
tstrncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN);
stub.handle = *pHandle;
++stub.refCount;
stub.createTime = taosGetTimestampUs();
@ -1129,7 +1131,7 @@ _exit:
void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) {
uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
tstrncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ);
if (!foundStub && !expiredStub) {
@ -2020,7 +2022,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
task->type = UDF_TASK_SETUP;
SUdfSetupRequest *req = &task->_setup.req;
strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN);
tstrncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN);
code = udfcRunUdfUvTask(task, UV_TASK_CONNECT);
TAOS_CHECK_GOTO(code, &lino, _exit);
@ -2033,7 +2035,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
task->session->outputType = rsp->outputType;
task->session->bytes = rsp->bytes;
task->session->bufSize = rsp->bufSize;
strncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN);
tstrncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN);
fnInfo("successfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session);
*funcHandle = task->session;
taosMemoryFree(task);

10
source/libs/function/src/udfd.c
View File

@ -396,7 +396,7 @@ int32_t udfdLoadSharedLib(char *libPath, uv_lib_t *pLib, const char *funcName[],
int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
plugin->scriptType = TSDB_FUNC_SCRIPT_PYTHON;
// todo: windows support
sprintf(plugin->libPath, "%s", "libtaospyudf.so");
snprintf(plugin->libPath, PATH_MAX, "%s", "libtaospyudf.so");
plugin->libLoaded = false;
const char *funcName[UDFD_MAX_PLUGIN_FUNCS] = {"pyOpen", "pyClose", "pyUdfInit",
"pyUdfDestroy", "pyUdfScalarProc", "pyUdfAggStart",
@ -617,7 +617,7 @@ int32_t udfdNewUdf(SUdf **pUdf, const char *udfName) {
}
udfNew->refCount = 1;
udfNew->lastFetchTime = taosGetTimestampMs();
strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN);
tstrncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN);
udfNew->state = UDF_STATE_INIT;
if (uv_mutex_init(&udfNew->lock) != 0) return TSDB_CODE_UDF_UV_EXEC_FAILURE;
@ -997,7 +997,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
udfdGetFuncBodyPath(udf, path);
bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0);
if (fileExist) {
strncpy(udf->path, path, PATH_MAX);
tstrncpy(udf->path, path, PATH_MAX);
fnInfo("udfd func body file. reuse existing file %s", path);
return TSDB_CODE_SUCCESS;
}
@ -1017,7 +1017,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
return TSDB_CODE_FILE_CORRUPTED;
}
strncpy(udf->path, path, PATH_MAX);
tstrncpy(udf->path, path, PATH_MAX);
return TSDB_CODE_SUCCESS;
}
@ -1612,7 +1612,7 @@ int32_t udfdInitResidentFuncs() {
char *token;
while ((token = strtok_r(pSave, ",", &pSave)) != NULL) {
char func[TSDB_FUNC_NAME_LEN + 1] = {0};
strncpy(func, token, TSDB_FUNC_NAME_LEN);
tstrncpy(func, token, TSDB_FUNC_NAME_LEN);
fnInfo("udfd add resident function %s", func);
if(taosArrayPush(global.residentFuncs, func) == NULL)
{

4
source/libs/parser/inc/parAst.h
View File

@ -154,6 +154,7 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode*
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createAnomalyWindowNode(SAstCreateContext* pCxt, SNode* pExpr, const SToken* pFuncOpt);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
@ -251,6 +252,9 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createCreateAnodeStmt(SAstCreateContext* pCxt, const SToken* pUrl);
SNode* createDropAnodeStmt(SAstCreateContext* pCxt, const SToken* pAnode);
SNode* createUpdateAnodeStmt(SAstCreateContext* pCxt, const SToken* pAnode, bool updateAll);
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue);
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,

60
source/libs/parser/src/parAstCreater.c
View File

@ -1367,6 +1367,25 @@ _err:
return NULL;
}
SNode* createAnomalyWindowNode(SAstCreateContext* pCxt, SNode* pExpr, const SToken* pFuncOpt) {
SAnomalyWindowNode* pAnomaly = NULL;
CHECK_PARSER_STATUS(pCxt);
pCxt->errCode = nodesMakeNode(QUERY_NODE_ANOMALY_WINDOW, (SNode**)&pAnomaly);
CHECK_MAKE_NODE(pAnomaly);
pAnomaly->pCol = createPrimaryKeyCol(pCxt, NULL);
CHECK_MAKE_NODE(pAnomaly->pCol);
pAnomaly->pExpr = pExpr;
if (pFuncOpt == NULL) {
tstrncpy(pAnomaly->anomalyOpt, "algo=iqr", TSDB_ANAL_ALGO_OPTION_LEN);
} else {
(void)trimString(pFuncOpt->z, pFuncOpt->n, pAnomaly->anomalyOpt, sizeof(pAnomaly->anomalyOpt));
}
return (SNode*)pAnomaly;
_err:
nodesDestroyNode((SNode*)pAnomaly);
return NULL;
}
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill) {
SIntervalWindowNode* interval = NULL;
@ -2997,6 +3016,47 @@ _err:
return NULL;
}
SNode* createCreateAnodeStmt(SAstCreateContext* pCxt, const SToken* pUrl) {
CHECK_PARSER_STATUS(pCxt);
SCreateAnodeStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_ANODE_STMT, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
(void)trimString(pUrl->z, pUrl->n, pStmt->url, sizeof(pStmt->url));
return (SNode*)pStmt;
_err:
return NULL;
}
SNode* createDropAnodeStmt(SAstCreateContext* pCxt, const SToken* pAnode) {
CHECK_PARSER_STATUS(pCxt);
SUpdateAnodeStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_ANODE_STMT, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
if (NULL != pAnode) {
pStmt->anodeId = taosStr2Int32(pAnode->z, NULL, 10);
} else {
pStmt->anodeId = -1;
}
return (SNode*)pStmt;
_err:
return NULL;
}
SNode* createUpdateAnodeStmt(SAstCreateContext* pCxt, const SToken* pAnode, bool updateAll) {
CHECK_PARSER_STATUS(pCxt);
SUpdateAnodeStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_UPDATE_ANODE_STMT, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
if (NULL != pAnode) {
pStmt->anodeId = taosStr2Int32(pAnode->z, NULL, 10);
} else {
pStmt->anodeId = -1;
}
return (SNode*)pStmt;
_err:
return NULL;
}
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue) {
SToken config;
config.type = TK_NK_STRING;

6
source/libs/parser/src/parInsertSml.c
View File

@ -113,7 +113,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
SSmlKv* kv = taosArrayGet(cols, i);
if (kv == NULL){
code = TSDB_CODE_SML_INVALID_DATA;
code = terrno;
uError("SML smlBuildTagRow error kv is null");
goto end;
}
@ -381,7 +381,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
for (int32_t r = 0; r < rowNum; ++r) {
void* rowData = taosArrayGetP(cols, r);
if (rowData == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
ret = terrno;
goto end;
}
// 1. set the parsed value from sql string
@ -389,7 +389,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
if (pVal == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
ret = terrno;
goto end;
}
void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));

2
source/libs/parser/src/parUtil.c
View File

@ -185,6 +185,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "%s is not supported in system table query";
case TSDB_CODE_PAR_INVALID_INTERP_CLAUSE:
return "Invalid usage of RANGE clause, EVERY clause or FILL clause";
case TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE:
return "Invalid usage of forecast clause";
case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN:
return "No valid function in window query";
case TSDB_CODE_PAR_INVALID_OPTR_USAGE:

2
source/libs/planner/src/planOptimizer.c
View File

@ -2380,6 +2380,8 @@ static bool sortPriKeyOptHasUnsupportedPkFunc(SLogicNode* pLogicNode, EOrder sor
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
pFuncList = ((SInterpFuncLogicNode*)pLogicNode)->pFuncs;
break;
case QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC:
pFuncList = ((SForecastFuncLogicNode*)pLogicNode)->pFuncs;
default:
break;
}

17
source/libs/planner/src/planSpliter.c
View File

@ -939,6 +939,18 @@ static int32_t stbSplSplitCount(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
}
}
static int32_t stbSplSplitAnomalyForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
static int32_t stbSplSplitAnomaly(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
if (pCxt->pPlanCxt->streamQuery) {
return stbSplSplitAnomalyForStream(pCxt, pInfo);
} else {
return stbSplSplitSessionOrStateForBatch(pCxt, pInfo);
}
}
static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) {
case WINDOW_TYPE_INTERVAL:
@ -951,6 +963,8 @@ static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitI
return stbSplSplitEvent(pCxt, pInfo);
case WINDOW_TYPE_COUNT:
return stbSplSplitCount(pCxt, pInfo);
case WINDOW_TYPE_ANOMALY:
return stbSplSplitAnomaly(pCxt, pInfo);
default:
break;
}
@ -2000,7 +2014,8 @@ typedef struct SQnodeSplitInfo {
static bool qndSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode,
SQnodeSplitInfo* pInfo) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != pNode->pParent &&
QUERY_NODE_LOGIC_PLAN_INTERP_FUNC != nodeType(pNode->pParent) && ((SScanLogicNode*)pNode)->scanSeq[0] <= 1 &&
QUERY_NODE_LOGIC_PLAN_INTERP_FUNC != nodeType(pNode->pParent) &&
QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC != nodeType(pNode->pParent) && ((SScanLogicNode*)pNode)->scanSeq[0] <= 1 &&
((SScanLogicNode*)pNode)->scanSeq[1] <= 1) {
pInfo->pSplitNode = pNode;
pInfo->pSubplan = pSubplan;

27
source/libs/qcom/src/queryUtil.c
View File

@ -313,42 +313,41 @@ void destroyQueryExecRes(SExecResult* pRes) {
}
}
// clang-format on
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) {
int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len) {
int32_t n = 0;
switch (type) {
case TSDB_DATA_TYPE_NULL:
n = sprintf(str, "null");
n = snprintf(str, capacity, "null");
break;
case TSDB_DATA_TYPE_BOOL:
n = sprintf(str, (*(int8_t*)buf) ? "true" : "false");
n = snprintf(str, capacity, (*(int8_t*)buf) ? "true" : "false");
break;
case TSDB_DATA_TYPE_TINYINT:
n = sprintf(str, "%d", *(int8_t*)buf);
n = snprintf(str, capacity, "%d", *(int8_t*)buf);
break;
case TSDB_DATA_TYPE_SMALLINT:
n = sprintf(str, "%d", *(int16_t*)buf);
n = snprintf(str, capacity, "%d", *(int16_t*)buf);
break;
case TSDB_DATA_TYPE_INT:
n = sprintf(str, "%d", *(int32_t*)buf);
n = snprintf(str, capacity, "%d", *(int32_t*)buf);
break;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
n = sprintf(str, "%" PRId64, *(int64_t*)buf);
n = snprintf(str, capacity, "%" PRId64, *(int64_t*)buf);
break;
case TSDB_DATA_TYPE_FLOAT:
n = sprintf(str, "%e", GET_FLOAT_VAL(buf));
n = snprintf(str, capacity, "%e", GET_FLOAT_VAL(buf));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = sprintf(str, "%e", GET_DOUBLE_VAL(buf));
n = snprintf(str, capacity, "%e", GET_DOUBLE_VAL(buf));
break;
case TSDB_DATA_TYPE_VARBINARY: {
@ -395,19 +394,19 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t
n = length + 2;
break;
case TSDB_DATA_TYPE_UTINYINT:
n = sprintf(str, "%d", *(uint8_t*)buf);
n = snprintf(str, capacity, "%d", *(uint8_t*)buf);
break;
case TSDB_DATA_TYPE_USMALLINT:
n = sprintf(str, "%d", *(uint16_t*)buf);
n = snprintf(str, capacity, "%d", *(uint16_t*)buf);
break;
case TSDB_DATA_TYPE_UINT:
n = sprintf(str, "%u", *(uint32_t*)buf);
n = snprintf(str, capacity, "%u", *(uint32_t*)buf);
break;
case TSDB_DATA_TYPE_UBIGINT:
n = sprintf(str, "%" PRIu64, *(uint64_t*)buf);
n = snprintf(str, capacity, "%" PRIu64, *(uint64_t*)buf);
break;
default:

36
source/libs/qcom/src/querymsg.c
View File

@ -107,7 +107,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms
}
SUseDbReq usedbReq = {0};
strncpy(usedbReq.db, pInput->db, sizeof(usedbReq.db));
tstrncpy(usedbReq.db, pInput->db, TSDB_DB_FNAME_LEN);
usedbReq.db[sizeof(usedbReq.db) - 1] = 0;
usedbReq.vgVersion = pInput->vgVersion;
usedbReq.dbId = pInput->dbId;
@ -207,7 +207,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t
}
SDbCfgReq dbCfgReq = {0};
strncpy(dbCfgReq.db, input, sizeof(dbCfgReq.db) - 1);
tstrncpy(dbCfgReq.db, input, TSDB_DB_FNAME_LEN);
int32_t bufLen = tSerializeSDbCfgReq(NULL, 0, &dbCfgReq);
void *pBuf = (*mallcFp)(bufLen);
@ -231,7 +231,7 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t
}
SUserIndexReq indexReq = {0};
strncpy(indexReq.indexFName, input, sizeof(indexReq.indexFName) - 1);
tstrncpy(indexReq.indexFName, input, TSDB_INDEX_FNAME_LEN);
int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq);
void *pBuf = (*mallcFp)(bufLen);
@ -293,7 +293,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32
}
SGetUserAuthReq req = {0};
strncpy(req.user, input, sizeof(req.user) - 1);
tstrncpy(req.user, input, TSDB_USER_LEN);
int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req);
void *pBuf = (*mallcFp)(bufLen);
@ -316,7 +316,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_
}
STableIndexReq indexReq = {0};
strncpy(indexReq.tbFName, input, sizeof(indexReq.tbFName) - 1);
tstrncpy(indexReq.tbFName, input, TSDB_TABLE_FNAME_LEN);
int32_t bufLen = tSerializeSTableIndexReq(NULL, 0, &indexReq);
void *pBuf = (*mallcFp)(bufLen);
@ -342,8 +342,8 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t
SBuildTableInput *pInput = input;
STableCfgReq cfgReq = {0};
cfgReq.header.vgId = pInput->vgId;
strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName) - 1);
strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName) - 1);
tstrncpy(cfgReq.dbFName, pInput->dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(cfgReq.tbName, pInput->tbName, TSDB_TABLE_NAME_LEN);
int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq);
void *pBuf = (*mallcFp)(bufLen);
@ -367,7 +367,7 @@ int32_t queryBuildGetViewMetaMsg(void *input, char **msg, int32_t msgSize, int32
}
SViewMetaReq req = {0};
strncpy(req.fullname, input, sizeof(req.fullname) - 1);
tstrncpy(req.fullname, input, TSDB_VIEW_FNAME_LEN);
int32_t bufLen = tSerializeSViewMetaReq(NULL, 0, &req);
void *pBuf = (*mallcFp)(bufLen);
@ -392,7 +392,7 @@ int32_t queryBuildGetTableTSMAMsg(void *input, char **msg, int32_t msgSize, int3
}
STableTSMAInfoReq req = {0};
strncpy(req.name, input, sizeof(req.name) - 1);
tstrncpy(req.name, input, TSDB_TABLE_FNAME_LEN);
int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req);
void * pBuf = (*mallcFp)(bufLen);
@ -417,7 +417,7 @@ int32_t queryBuildGetTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t *
STableTSMAInfoReq req = {0};
req.fetchingWithTsmaName = true;
strncpy(req.name, input, sizeof(req.name) - 1);
tstrncpy(req.name, input, TSDB_TABLE_FNAME_LEN);
int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req);
void * pBuf = (*mallcFp)(bufLen);
@ -688,14 +688,14 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) {
}
STableMetaOutput *pOut = output;
strcpy(pOut->dbFName, metaRsp.dbFName);
tstrncpy(pOut->dbFName, metaRsp.dbFName, TSDB_DB_FNAME_LEN);
pOut->dbId = metaRsp.dbId;
if (metaRsp.tableType == TSDB_CHILD_TABLE) {
SET_META_TYPE_BOTH_TABLE(pOut->metaType);
strcpy(pOut->ctbName, metaRsp.tbName);
strcpy(pOut->tbName, metaRsp.stbName);
tstrncpy(pOut->ctbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN);
tstrncpy(pOut->tbName, metaRsp.stbName, TSDB_TABLE_NAME_LEN);
pOut->ctbMeta.vgId = metaRsp.vgId;
pOut->ctbMeta.tableType = metaRsp.tableType;
@ -705,7 +705,7 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) {
code = queryCreateTableMetaFromMsg(&metaRsp, true, &pOut->tbMeta);
} else {
SET_META_TYPE_TABLE(pOut->metaType);
strcpy(pOut->tbName, metaRsp.tbName);
tstrncpy(pOut->tbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN);
code = queryCreateTableMetaFromMsg(&metaRsp, (metaRsp.tableType == TSDB_SUPER_TABLE), &pOut->tbMeta);
}
@ -744,14 +744,14 @@ static int32_t queryProcessTableNameRsp(void *output, char *msg, int32_t msgSize
}
STableMetaOutput *pOut = output;
strcpy(pOut->dbFName, metaRsp.dbFName);
tstrncpy(pOut->dbFName, metaRsp.dbFName, TSDB_DB_FNAME_LEN);
pOut->dbId = metaRsp.dbId;
if (metaRsp.tableType == TSDB_CHILD_TABLE) {
SET_META_TYPE_BOTH_TABLE(pOut->metaType);
strcpy(pOut->ctbName, metaRsp.tbName);
strcpy(pOut->tbName, metaRsp.stbName);
tstrncpy(pOut->ctbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN);
tstrncpy(pOut->tbName, metaRsp.stbName, TSDB_TABLE_NAME_LEN);
pOut->ctbMeta.vgId = metaRsp.vgId;
pOut->ctbMeta.tableType = metaRsp.tableType;
@ -761,7 +761,7 @@ static int32_t queryProcessTableNameRsp(void *output, char *msg, int32_t msgSize
code = queryCreateTableMetaExFromMsg(&metaRsp, true, &pOut->tbMeta);
} else {
SET_META_TYPE_TABLE(pOut->metaType);
strcpy(pOut->tbName, metaRsp.tbName);
tstrncpy(pOut->tbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN);
code = queryCreateTableMetaExFromMsg(&metaRsp, (metaRsp.tableType == TSDB_SUPER_TABLE), &pOut->tbMeta);
}

89
source/libs/scalar/src/filter.c
View File

@ -1764,41 +1764,41 @@ _return:
return DEAL_RES_ERROR;
}
int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t fltConverToStr(char *str, int32_t strMaxLen, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t n = 0;
switch (type) {
case TSDB_DATA_TYPE_NULL:
n = sprintf(str, "null");
n = snprintf(str, strMaxLen, "null");
break;
case TSDB_DATA_TYPE_BOOL:
n = sprintf(str, (*(int8_t *)buf) ? "true" : "false");
n = snprintf(str, strMaxLen, (*(int8_t *)buf) ? "true" : "false");
break;
case TSDB_DATA_TYPE_TINYINT:
n = sprintf(str, "%d", *(int8_t *)buf);
n = snprintf(str, strMaxLen, "%d", *(int8_t *)buf);
break;
case TSDB_DATA_TYPE_SMALLINT:
n = sprintf(str, "%d", *(int16_t *)buf);
n = snprintf(str, strMaxLen, "%d", *(int16_t *)buf);
break;
case TSDB_DATA_TYPE_INT:
n = sprintf(str, "%d", *(int32_t *)buf);
n = snprintf(str, strMaxLen, "%d", *(int32_t *)buf);
break;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
n = sprintf(str, "%" PRId64, *(int64_t *)buf);
n = snprintf(str, strMaxLen, "%" PRId64, *(int64_t *)buf);
break;
case TSDB_DATA_TYPE_FLOAT:
n = sprintf(str, "%e", GET_FLOAT_VAL(buf));
n = snprintf(str, strMaxLen, "%e", GET_FLOAT_VAL(buf));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = sprintf(str, "%e", GET_DOUBLE_VAL(buf));
n = snprintf(str, strMaxLen, "%e", GET_DOUBLE_VAL(buf));
break;
case TSDB_DATA_TYPE_BINARY:
@ -1817,19 +1817,19 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
break;
case TSDB_DATA_TYPE_UTINYINT:
n = sprintf(str, "%d", *(uint8_t *)buf);
n = snprintf(str, strMaxLen, "%d", *(uint8_t *)buf);
break;
case TSDB_DATA_TYPE_USMALLINT:
n = sprintf(str, "%d", *(uint16_t *)buf);
n = snprintf(str, strMaxLen, "%d", *(uint16_t *)buf);
break;
case TSDB_DATA_TYPE_UINT:
n = sprintf(str, "%u", *(uint32_t *)buf);
n = snprintf(str, strMaxLen, "%u", *(uint32_t *)buf);
break;
case TSDB_DATA_TYPE_UBIGINT:
n = sprintf(str, "%" PRIu64, *(uint64_t *)buf);
n = snprintf(str, strMaxLen, "%" PRIu64, *(uint64_t *)buf);
break;
default:
@ -1886,7 +1886,7 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SColumnNode *refNode = (SColumnNode *)left->desc;
if (unit->compare.optr <= OP_TYPE_JSON_CONTAINS) {
len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId,
len += snprintf(str, sizeof(str), "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId,
operatorTypeStr(unit->compare.optr));
}
@ -1898,18 +1898,22 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio
data += VARSTR_HEADER_SIZE;
}
if (data) {
FLT_ERR_RET(fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen));
FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, type, data, tlen > 32 ? 32 : tlen, &tlen));
len += tlen;
}
} else {
(void)strcat(str, "NULL");
(void)strncat(str, "NULL", sizeof(str) - len - 1);
len += 4;
}
(void)strcat(str, "]");
(void)strncat(str, "]", sizeof(str) - len - 1);
len += 1;
if (unit->compare.optr2) {
(void)strcat(str, " && ");
(void)strncat(str, " && ", sizeof(str) - len - 1);
len += 4;
if (unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) {
(void)sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId,
operatorTypeStr(unit->compare.optr2));
len += snprintf(str + len, sizeof(str) - len, "[%d][%d] %s [", refNode->dataBlockId,
refNode->slotId, operatorTypeStr(unit->compare.optr2));
}
if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) {
@ -1919,11 +1923,14 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio
tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE;
}
FLT_ERR_RET(fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen));
FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, type, data, tlen > 32 ? 32 : tlen, &tlen));
len += tlen;
} else {
(void)strcat(str, "NULL");
(void)strncat(str, "NULL", sizeof(str) - len - 1);
len += 4;
}
(void)strcat(str, "]");
(void)strncat(str, "]", sizeof(str) - len - 1);
len += 1;
}
qDebug("%s", str); // TODO
@ -1956,20 +1963,38 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio
int32_t tlen = 0;
while (r) {
char str[256] = {0};
int32_t len = 0;
if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) {
(void)strcat(str, "(NULL)");
(void)strncat(str, "(NULL)", sizeof(str) - len - 1);
len += 6;
} else {
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str, "(") : strcat(str, "[");
FLT_ERR_RET(fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen));
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str, ")") : strcat(str, "]");
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ?
(void)strncat(str, "(", sizeof(str) - len - 1) :
(void)strncat(str, "[", sizeof(str) - len - 1);
len += 1;
FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen));
len += tlen;
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ?
(void)strncat(str, ")", sizeof(str) - len - 1) :
(void)strncat(str, "]", sizeof(str) - len - 1);
len += 1;
}
(void)strcat(str, " - ");
(void)strncat(str, " - ", sizeof(str) - len - 1);
len += 3;
if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) {
(void)strcat(str, "(NULL)");
(void)strncat(str, "(NULL)", sizeof(str) - len - 1);
len += 6;
} else {
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str, "(") : strcat(str, "[");
FLT_ERR_RET(fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen));
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str, ")") : strcat(str, "]");
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ?
(void)strncat(str, "(", sizeof(str) - len - 1) :
(void)strncat(str, "[", sizeof(str) - len - 1);
len += 1;
FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen));
len += tlen;
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ?
(void)strncat(str, ")", sizeof(str) - len - 1) :
(void)strncat(str, "]", sizeof(str) - len - 1);
len += 1;
}
qDebug("range: %s", str);

8
source/libs/scalar/src/scalar.c
View File

@ -1211,7 +1211,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) {
res->translate = true;
(void)strcpy(res->node.aliasName, node->node.aliasName);
tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN);
res->node.resType.type = output.columnData->info.type;
res->node.resType.bytes = output.columnData->info.bytes;
res->node.resType.scale = output.columnData->info.scale;
@ -1286,7 +1286,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
res->node.resType = node->node.resType;
res->translate = true;
(void)strcpy(res->node.aliasName, node->node.aliasName);
tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN);
int32_t type = output.columnData->info.type;
if (IS_VAR_DATA_TYPE(type)) {
res->datum.p = output.columnData->pData;
@ -1356,7 +1356,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
res->translate = true;
(void)strcpy(res->node.aliasName, node->node.aliasName);
tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN);
res->node.resType = node->node.resType;
if (colDataIsNull_s(output.columnData, 0)) {
res->isNull = true;
@ -1419,7 +1419,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) {
res->translate = true;
(void)strcpy(res->node.aliasName, node->node.aliasName);
tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN);
res->node.resType = node->node.resType;
if (colDataIsNull_s(output.columnData, 0)) {
res->isNull = true;

20
source/libs/scalar/src/sclfunc.c
View File

@ -2067,9 +2067,9 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
if (inputType == TSDB_DATA_TYPE_BOOL) {
// NOTE: sprintf will append '\0' at the end of string
int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE),
*(int8_t *)input ? "true" : "false");
// NOTE: snprintf will append '\0' at the end of string
int32_t len = snprintf(varDataVal(output), outputLen + TSDB_NCHAR_SIZE - VARSTR_HEADER_SIZE, "%.*s",
(int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false");
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_BINARY) {
int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE);
@ -2109,7 +2109,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int32_t len;
if (inputType == TSDB_DATA_TYPE_BOOL) {
char tmp[8] = {0};
len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false");
len = snprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false");
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -3972,6 +3972,10 @@ int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
return nonCalcScalarFunction(pInput, inputNum, pOutput);
}
int32_t forecastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return nonCalcScalarFunction(pInput, inputNum, pOutput);
}
int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return avgScalarFunction(pInput, inputNum, pOutput);
}
@ -4407,11 +4411,11 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
int32_t len;
char buf[512] = {0};
if (!normalized) {
len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", bins[k].lower,
bins[k].upper, bins[k].count);
len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}",
bins[k].lower, bins[k].upper, bins[k].count);
} else {
len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", bins[k].lower,
bins[k].upper, bins[k].percentage);
len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}",
bins[k].lower, bins[k].upper, bins[k].percentage);
}
varDataSetLen(buf, len);
SCL_ERR_JRET(colDataSetVal(pOutputData, k, buf, false));

6
source/libs/scalar/src/sclvector.c
View File

@ -734,7 +734,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) {
int64_t value = 0;
GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value);
int32_t len = snprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%" PRId64, value);
varDataLen(tmp) = len;
if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) {
SCL_ERR_RET(varToNchar(tmp, pCtx->pOut, i, NULL));
@ -751,7 +751,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) {
uint64_t value = 0;
GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value);
int32_t len = snprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%" PRIu64, value);
varDataLen(tmp) = len;
if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) {
SCL_ERR_RET(varToNchar(tmp, pCtx->pOut, i, NULL));
@ -768,7 +768,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) {
double value = 0;
GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%lf", value);
int32_t len = snprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%lf", value);
varDataLen(tmp) = len;
if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) {
SCL_ERR_RET(varToNchar(tmp, pCtx->pOut, i, NULL));

6
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -55,7 +55,7 @@ void flttInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
(void)strcpy(tsLogDir, TD_LOG_DIR_PATH);
tstrncpy(tsLogDir, TD_LOG_DIR_PATH, PATH_MAX);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
@ -101,7 +101,7 @@ int32_t flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType,
rnode->node.resType.bytes = dataBytes;
rnode->dataBlockId = 0;
sprintf(rnode->dbName, "%" PRIu64, dbidx++);
snprintf(rnode->dbName, TSDB_DB_NAME_LEN, "%" PRIu64, dbidx++);
if (NULL == block) {
rnode->slotId = 2;
@ -666,7 +666,7 @@ TEST(columnTest, binary_column_like_binary) {
int32_t rowNum = sizeof(leftv) / sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
sprintf(&rightv[2], "%s", "__0");
snprintf(&rightv[2], sizeof(rightv) - 2, "%s", "__0");
varDataSetLen(rightv, strlen(&rightv[2]));
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv);
flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight);

2
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -81,7 +81,7 @@ void scltInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
(void)strcpy(tsLogDir, TD_LOG_DIR_PATH);
tstrncpy(tsLogDir, TD_LOG_DIR_PATH, PATH_MAX);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) {
(void)printf("failed to open log file in directory:%s\n", tsLogDir);

4
source/libs/tdb/src/db/tdbPage.c
View File

@ -102,6 +102,10 @@ void tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg
tdbOsFree(pPage->apOvfl[iOvfl]);
}
if (TDB_DESTROY_PAGE_LOCK(pPage) != 0) {
tdbError("tdb/page-destroy: destroy page lock failed.");
}
ptr = pPage->pData;
xFree(arg, ptr);

28
source/os/src/osSemaphore.c
View File

@ -377,20 +377,20 @@ int32_t tsem2_wait(tsem2_t* sem) {
}
int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) {
int ret = 0;
int32_t code = 0;
ret = taosThreadMutexLock(&sem->mutex);
if (ret) {
return ret;
code = taosThreadMutexLock(&sem->mutex);
if (code) {
return code;
}
if (sem->count <= 0) {
struct timespec ts = {0};
if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) {
ret = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
(void)taosThreadMutexUnlock(&sem->mutex);
terrno = ret;
return ret;
terrno = code;
return code;
}
ts.tv_sec += ms / 1000;
@ -399,22 +399,18 @@ int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) {
ts.tv_nsec %= 1000000000;
while (sem->count <= 0) {
ret = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts);
if (ret != 0) {
code = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts);
if (code != 0) {
(void)taosThreadMutexUnlock(&sem->mutex);
if (errno == ETIMEDOUT) {
return TSDB_CODE_TIMEOUT_ERROR;
} else {
return TAOS_SYSTEM_ERROR(errno);
}
return code;
}
}
}
sem->count--;
ret = taosThreadMutexUnlock(&sem->mutex);
return ret;
code = taosThreadMutexUnlock(&sem->mutex);
return code;
}
#endif

Some files were not shown because too many files have changed in this diff Show More