diff --git a/Jenkinsfile2 b/Jenkinsfile2
index b65576deaf..f618912295 100644
--- a/Jenkinsfile2
+++ b/Jenkinsfile2
@@ -4,11 +4,6 @@ import jenkins.model.CauseOfInterruption
node {
}
-win_test_stage = 0
-linux_ready = 0
-linux_node_ip = ""
-linux_node_pass = ""
-
def abortPreviousBuilds() {
def currentJobName = env.JOB_NAME
def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -84,6 +79,7 @@ def pre_test(){
git pull >/dev/null
git log -5
echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
+ echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
@@ -100,6 +96,7 @@ def pre_test(){
git pull >/dev/null
git log -5
echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
+ echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
@@ -289,7 +286,6 @@ def run_win_ctest() {
'''
}
def run_win_test() {
- echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}"
bat '''
echo "windows test ..."
cd %WIN_CONNECTOR_ROOT%
@@ -298,9 +294,8 @@ def run_win_test() {
ls -l C:\\Windows\\System32\\taos.dll
time /t
cd %WIN_SYSTEM_TEST_ROOT%
- echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''"
echo "testing ..."
- test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}"
+ test-all.bat ci
time /t
'''
}
@@ -331,17 +326,9 @@ pipeline {
pre_test_win()
pre_test_build_win()
run_win_ctest()
- script {
- while(linux_ready == 0) {
- sleep(8)
- }
- }
run_win_test()
}
}
- script {
- win_test_stage = 1
- }
}
}
stage('linux test') {
@@ -352,15 +339,12 @@ pipeline {
}
steps {
script {
- linux_node_ip = sh (
- script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"',
+ def linux_node_ip = sh (
+ script: 'ip addr|grep 192|grep -v virbr|awk "{print \\\$2}"|sed "s/\\/.*//"',
returnStdout: true
).trim()
- linux_node_pass = sh (
- script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"',
- returnStdout: true
- ).trim()
- echo "${linux_node_ip}:${linux_node_pass}"
+ echo "${linux_node_ip}"
+ echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
}
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 120, unit: 'MINUTES'){
@@ -418,38 +402,9 @@ pipeline {
cd ${WKC}/packaging
./release.sh -v cluster -n 3.0.0.100 -s static
'''
- sh '''
- echo "install ..."
- cd ${WKC}/release
- tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz
- cd TDengine-enterprise-server-3.0.0.100
- service taosd stop || :
- rm -rf /var/lib/taos
- ./install.sh -e no
- '''
- sh '''
- echo "checking ..."
- which taos
- which taosd
- rm -rf ${WK}/debug
- mv ${WKC}/debug ${WK}/
- '''
- sh '''
- echo "install taospy ..."
- cd ${WKPY}
- pip3 install .
- '''
}
}
}
- script {
- linux_ready = 1
- }
- script {
- while(win_test_stage == 0){
- sleep(12)
- }
- }
}
}
}
diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md
index d7d472eb98..44651c0496 100644
--- a/docs/en/21-tdinternal/01-arch.md
+++ b/docs/en/21-tdinternal/01-arch.md
@@ -22,7 +22,7 @@ A complete TDengine system runs on one or more physical nodes. Logically, it inc
**Virtual node (vnode)**: To better support data sharding, load balancing and prevent data from overheating or skewing, data nodes are virtualized into multiple virtual nodes (vnode, V2, V3, V4, etc. in the figure). Each vnode is a relatively independent work unit, which is the basic unit of time-series data storage and has independent running threads, memory space and persistent storage path. A vnode contains a certain number of tables (data collection points). When a new table is created, the system checks whether a new vnode needs to be created. The number of vnodes that can be created on a data node depends on the capacity of the hardware of the physical node where the data node is located. A vnode belongs to only one DB, but a DB can have multiple vnodes. In addition to the stored time-series data, a vnode also stores the schema and tag values of the included tables. A virtual node is uniquely identified in the system by the EP of the data node and the VGroup ID to which it belongs and is created and managed by the management node.
-**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 5) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction.
+**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 3) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction.
**Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed in a leader/follower mechanism. Write operations can only be performed on the leader vnode, and then replicated to follower vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `“replica”` when creating a DB, and the default is 1. Using the multi-replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID. If two virtual nodes have the same vnode group ID, it means that they belong to the same group and the data is backed up to each other. The number of virtual nodes in a virtual node group can be dynamically changed, allowing only one, that is, no data replication. VGroup ID is never changed. Even if a virtual node group is deleted, its ID will not be reused.
diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md
index 433cb4808b..507ccea629 100644
--- a/docs/zh/21-tdinternal/01-arch.md
+++ b/docs/zh/21-tdinternal/01-arch.md
@@ -23,7 +23,7 @@ TDengine 分布式架构的逻辑结构图如下：
**虚拟节点（vnode）：** 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点（vnode，图中 V2，V3，V4 等）。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个 DB，但一个 DB 可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的 schema、标签值等。一个虚拟节点由所属的数据节点的 EP，以及所属的 VGroup ID 在系统内唯一标识，由管理节点创建并管理。
-**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（开源版最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。
+**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。
**虚拟节点组（VGroup）：** 不同数据节点上的 vnode 可以组成一个虚拟节点组（vgroup）来保证系统的高可靠。虚拟节点组内采取 master/slave 的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个 DB 的副本数为 N，系统必须有至少 N 数据节点。副本数在创建 DB 时通过参数 replica 可以指定，缺省为 1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的 ID，VGroup ID。如果两个虚拟节点的 VGroup ID 相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID 是永远不变的，即使一个虚拟节点组被删除，它的 ID 也不会被收回重复利用。
diff --git a/include/client/taos.h b/include/client/taos.h
index 5e7f12de0a..d31d5c582c 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -128,6 +128,7 @@ typedef struct setConfRet {
DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
+DLL_EXPORT setConfRet taos_set_config(const char *config);
DLL_EXPORT int taos_init(void);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 9c84d0dd99..928fe0aa0e 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -42,11 +42,13 @@ enum {
typedef enum EStreamType {
STREAM_NORMAL = 1,
STREAM_INVERT,
- STREAM_REPROCESS,
+ STREAM_CLEAR,
STREAM_INVALID,
STREAM_GET_ALL,
STREAM_DELETE,
STREAM_RETRIEVE,
+ STREAM_PUSH_DATA,
+ STREAM_PUSH_EMPTY,
} EStreamType;
typedef struct {
@@ -71,7 +73,7 @@ typedef struct SColumnDataAgg {
typedef struct SDataBlockInfo {
STimeWindow window;
- int32_t rows; // todo hide this attribute
+ int32_t rows; // todo hide this attribute
int32_t rowSize;
uint64_t uid; // the uid of table, from which current data block comes
uint16_t blockId; // block id, generated by physical planner
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 60f517c7ba..a792528fa8 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -225,6 +225,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
+int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createDataBlock();
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
@@ -236,7 +237,9 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t*
int8_t needCompress);
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
-void blockDebugShowData(const SArray* dataBlocks, const char* flag);
+void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag);
+// for debug
+char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf);
int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
tb_uid_t suid);
diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index f9ede63f7f..8f7f22a6a0 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -78,7 +78,6 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove
-void debugCheckTags(STag *pTag); // TODO: remove
// STRUCT =================
struct STColumn {
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index a5688af18a..c5b0b89311 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -1886,7 +1886,7 @@ typedef struct SVCreateStbReq {
int8_t rollup;
SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag;
- SRSmaParam pRSmaParam;
+ SRSmaParam rsmaParam;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index e23b84c1f8..0bd917a9c6 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -24,6 +24,8 @@ extern "C" {
#include "querynodes.h"
#include "tname.h"
+#define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN
+
typedef struct SLogicNode {
ENodeType type;
SNodeList* pTargets; // SColumnNode
@@ -74,8 +76,8 @@ typedef struct SScanLogicNode {
int16_t tsColId;
double filesFactor;
SArray* pSmaIndexes;
- SNodeList* pPartTags;
- bool partSort;
+ SNodeList* pGroupTags;
+ bool groupSort;
} SScanLogicNode;
typedef struct SJoinLogicNode {
@@ -100,6 +102,7 @@ typedef struct SProjectLogicNode {
typedef struct SIndefRowsFuncLogicNode {
SLogicNode node;
SNodeList* pFuncs;
+ bool isTailFunc;
} SIndefRowsFuncLogicNode;
typedef struct SInterpFuncLogicNode {
@@ -138,6 +141,7 @@ typedef struct SMergeLogicNode {
SNodeList* pInputs;
int32_t numOfChannels;
int32_t srcGroupId;
+ bool groupSort;
} SMergeLogicNode;
typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
@@ -184,6 +188,7 @@ typedef struct SFillLogicNode {
typedef struct SSortLogicNode {
SLogicNode node;
SNodeList* pSortKeys;
+ bool groupSort;
} SSortLogicNode;
typedef struct SPartitionLogicNode {
@@ -230,6 +235,7 @@ typedef struct SSlotDescNode {
bool reserve;
bool output;
bool tag;
+ char name[SLOT_NAME_LEN];
} SSlotDescNode;
typedef struct SDataBlockDescNode {
@@ -279,7 +285,8 @@ typedef struct STableScanPhysiNode {
double ratio;
int32_t dataRequired;
SNodeList* pDynamicScanFuncs;
- SNodeList* pPartitionTags;
+ SNodeList* pGroupTags;
+ bool groupSort;
int64_t interval;
int64_t offset;
int64_t sliding;
@@ -353,6 +360,7 @@ typedef struct SMergePhysiNode {
SNodeList* pTargets;
int32_t numOfChannels;
int32_t srcGroupId;
+ bool groupSort;
} SMergePhysiNode;
typedef struct SWinodwPhysiNode {
diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index a6e466e73e..39c569acef 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -259,6 +259,7 @@ typedef struct SSelectStmt {
bool hasTailFunc;
bool hasInterpFunc;
bool hasLastRowFunc;
+ bool groupSort;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index d86a8dae6e..0b767e96f6 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -16,6 +16,7 @@
#ifndef _TD_QUERY_H_
#define _TD_QUERY_H_
+// clang-foramt off
#ifdef __cplusplus
extern "C" {
#endif
@@ -71,7 +72,7 @@ typedef struct SIndexMeta {
} SIndexMeta;
typedef struct STbVerInfo {
- char tbFName[TSDB_TABLE_FNAME_LEN];
+ char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t sversion;
int32_t tversion;
} STbVerInfo;
@@ -141,7 +142,7 @@ typedef struct SDataBuf {
typedef struct STargetInfo {
ETargetType type;
- char* dbFName; // used to update db's vgroup epset
+ char* dbFName; // used to update db's vgroup epset
int32_t vgId;
} STargetInfo;
@@ -149,15 +150,15 @@ typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32
typedef int32_t (*__async_exec_fn_t)(void* param);
typedef struct SRequestConnInfo {
- void* pTrans;
- uint64_t requestId;
- int64_t requestObjRefId;
- SEpSet mgmtEps;
+ void* pTrans;
+ uint64_t requestId;
+ int64_t requestObjRefId;
+ SEpSet mgmtEps;
} SRequestConnInfo;
typedef struct SMsgSendInfo {
- __async_send_cb_fn_t fp; // async callback function
- STargetInfo target; // for update epset
+ __async_send_cb_fn_t fp; // async callback function
+ STargetInfo target; // for update epset
void* param;
uint64_t requestId;
uint64_t requestObjRefId;
@@ -206,13 +207,15 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
char* jobTaskStatusStr(int32_t status);
SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
-void destroyQueryExecRes(SQueryExecRes* pRes);
-int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len);
-char* parseTagDatatoJson(void* p);
+
+void destroyQueryExecRes(SQueryExecRes* pRes);
+int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
+char* parseTagDatatoJson(void* p);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);
-extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t));
+extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen,
+ void* (*mallocFp)(int32_t));
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
@@ -223,7 +226,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \
(_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
- (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
+ (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
(_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
@@ -231,11 +234,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_HANDLE_ERROR(_code) \
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
-#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \
- || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB)
+#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
+ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
+ (_type) == TDMT_VND_DROP_STB)
-#define NEED_SCHEDULER_RETRY_ERROR(_code) \
- ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+#define NEED_SCHEDULER_RETRY_ERROR(_code) \
+ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
+ (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
#define REQUEST_TOTAL_EXEC_TIMES 2
@@ -312,3 +317,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#endif
#endif /*_TD_QUERY_H_*/
+ // clang-foramt on
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index a47810e7c1..db928f194c 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -58,6 +58,7 @@ enum {
enum {
STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK,
+ STREAM_INPUT__DATA_RETRIEVE,
STREAM_INPUT__TRIGGER,
STREAM_INPUT__CHECKPOINT,
STREAM_INPUT__DROP,
@@ -318,7 +319,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
return -1;
}
taosWriteQitem(pTask->inputQueue->queue, pSubmitClone);
- } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+ } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
taosWriteQitem(pTask->inputQueue->queue, pItem);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
taosWriteQitem(pTask->inputQueue->queue, pItem);
diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h
index 398851a09f..21a1515d8f 100644
--- a/include/libs/stream/tstreamUpdate.h
+++ b/include/libs/stream/tstreamUpdate.h
@@ -32,12 +32,15 @@ typedef struct SUpdateInfo {
int64_t interval;
int64_t watermark;
TSKEY minTS;
+ SScalableBf* pCloseWinSBF;
} SUpdateInfo;
SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
void updateInfoDestroy(SUpdateInfo *pInfo);
+void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
+void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
#ifdef __cplusplus
}
diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index 52d9b6f810..5fa7eed40c 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -26,9 +26,9 @@ extern "C" {
extern bool gRaftDetailLog;
-#define SYNC_INDEX_BEGIN 0
+#define SYNC_INDEX_BEGIN 0
#define SYNC_INDEX_INVALID -1
-#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF
+#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF
typedef uint64_t SyncNodeId;
typedef int32_t SyncGroupId;
diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index e51b20a404..5d892352d6 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -524,7 +524,7 @@ void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg);
int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
-int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex);
int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
@@ -541,7 +541,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode* ths, SyncSnapshotRsp* pMsg);
// -----------------------------------------
typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg);
typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg);
-typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg);
+typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex);
typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg);
typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg);
typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg);
diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 77de5f4756..c2c1a3534d 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -34,10 +34,8 @@ extern int32_t tsRpcHeadSize;
typedef struct {
uint32_t clientIp;
uint16_t clientPort;
- union {
- char user[TSDB_USER_LEN];
- int64_t applyIndex;
- };
+ int64_t applyIndex;
+ char user[TSDB_USER_LEN];
} SRpcConnInfo;
typedef struct SRpcHandleInfo {
diff --git a/include/os/osSocket.h b/include/os/osSocket.h
index 213a6930ee..9dd5b972fa 100644
--- a/include/os/osSocket.h
+++ b/include/os/osSocket.h
@@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
void taosWinSocketInit();
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
+/*
+ * set timeout(ms)
+ */
+int32_t taosCreateSocketWithTimeout(uint32_t timeout);
TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
diff --git a/include/util/ttrace.h b/include/util/ttrace.h
index 206cbbf28d..579768228a 100644
--- a/include/util/ttrace.h
+++ b/include/util/ttrace.h
@@ -45,9 +45,11 @@ typedef struct STraceId {
#define TRACE_GET_MSGID(traceId) (traceId)->msgId
-#define TRACE_TO_STR(traceId, buf) \
- do { \
- sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", traceId->rootId, traceId->msgId); \
+#define TRACE_TO_STR(traceId, buf) \
+ do { \
+ int64_t rootId = (traceId) != NULL ? (traceId)->rootId : 0; \
+ int64_t msgId = (traceId) != NULL ? (traceId)->msgId : 0; \
+ sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", rootId, msgId); \
} while (0)
#ifdef __cplusplus
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 9f04e89694..d7bf4b60f1 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -13,11 +13,11 @@
* along with this program. If not, see .
*/
-#include "os.h"
#include "catalog.h"
-#include "functionMgt.h"
#include "clientInt.h"
#include "clientLog.h"
+#include "functionMgt.h"
+#include "os.h"
#include "query.h"
#include "scheduler.h"
#include "tcache.h"
@@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
volatile int32_t tscInitRes = 0;
static void registerRequest(SRequestObj *pRequest) {
- STscObj *pTscObj = acquireTscObj(*(int64_t*)pRequest->pTscObj->id);
+ STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id);
assert(pTscObj != NULL);
@@ -54,14 +54,14 @@ static void registerRequest(SRequestObj *pRequest) {
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
- pRequest->self, *(int64_t*)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
+ pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
}
}
static void deregisterRequest(SRequestObj *pRequest) {
assert(pRequest != NULL);
- STscObj *pTscObj = pRequest->pTscObj;
+ STscObj * pTscObj = pRequest->pTscObj;
SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
@@ -70,8 +70,8 @@ static void deregisterRequest(SRequestObj *pRequest) {
int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
" ms, current:%d, app current:%d",
- pRequest->self, *(int64_t*)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
- releaseTscObj(*(int64_t*)pTscObj->id);
+ pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
+ releaseTscObj(*(int64_t *)pTscObj->id);
}
// todo close the transporter properly
@@ -80,12 +80,13 @@ void closeTransporter(STscObj *pTscObj) {
return;
}
- tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t*)pTscObj->id);
+ tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id);
rpcClose(pTscObj->pAppInfo->pTransporter);
}
static bool clientRpcRfp(int32_t code) {
- if (code == TSDB_CODE_RPC_REDIRECT) {
+ if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
return true;
} else {
return false;
@@ -128,16 +129,17 @@ void closeAllRequests(SHashObj *pRequests) {
void destroyTscObj(void *pObj) {
STscObj *pTscObj = pObj;
- SClientHbKey connKey = {.tscRid = *(int64_t*)pTscObj->id, .connType = pTscObj->connType};
+ SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType};
hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
closeAllRequests(pTscObj->pRequests);
schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter);
if (0 == connNum) {
- // TODO
- //closeTransporter(pTscObj);
+ // TODO
+ // closeTransporter(pTscObj);
}
- tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t*)pTscObj->id, pTscObj->pAppInfo->numOfConns);
+ tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id,
+ pTscObj->pAppInfo->numOfConns);
taosThreadMutexDestroy(&pTscObj->mutex);
taosMemoryFreeClear(pTscObj);
}
@@ -167,10 +169,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
taosThreadMutexInit(&pObj->mutex, NULL);
pObj->id = taosMemoryMalloc(sizeof(int64_t));
- *(int64_t*)pObj->id = taosAddRef(clientConnRefPool, pObj);
+ *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj);
pObj->schemalessType = 1;
- tscDebug("connObj created, 0x%" PRIx64, *(int64_t*)pObj->id);
+ tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id);
return pObj;
}
@@ -325,7 +327,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
return 0;
}
- SConfig *pCfg = taosGetCfg();
+ SConfig * pCfg = taosGetCfg();
SConfigItem *pItem = NULL;
switch (option) {
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index f5dfe2de36..52574dcc9f 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -81,6 +81,19 @@ void taos_cleanup(void) {
taosCloseLog();
}
+static setConfRet taos_set_config_imp(const char *config){
+ setConfRet ret = {SET_CONF_RET_SUCC, {0}};
+ // TODO: need re-implementation
+ return ret;
+}
+
+setConfRet taos_set_config(const char *config){
+// TODO pthread_mutex_lock(&setConfMutex);
+ setConfRet ret = taos_set_config_imp(config);
+// pthread_mutex_unlock(&setConfMutex);
+ return ret;
+}
+
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db);
if (user == NULL) {
@@ -199,10 +212,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
return pResInfo->userFields;
}
-
-TAOS_RES *taos_query(TAOS *taos, const char *sql) {
- return taosQueryImpl(taos, sql, false);
-}
+TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); }
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
if (res == NULL) {
@@ -593,11 +603,11 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
return pResInfo->pCol[columnIndex].offset;
}
-int taos_validate_sql(TAOS *taos, const char *sql) {
- TAOS_RES* pObj = taosQueryImpl(taos, sql, true);
+int taos_validate_sql(TAOS *taos, const char *sql) {
+ TAOS_RES *pObj = taosQueryImpl(taos, sql, true);
int code = taos_errno(pObj);
-
+
taos_free_result(pObj);
return code;
}
@@ -884,10 +894,10 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
int taos_load_table_info(TAOS *taos, const char *tableNameList) {
const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list
- int32_t code = 0;
- SRequestObj *pRequest = NULL;
- SCatalogReq catalogReq = {0};
-
+ int32_t code = 0;
+ SRequestObj *pRequest = NULL;
+ SCatalogReq catalogReq = {0};
+
if (NULL == tableNameList) {
return TSDB_CODE_SUCCESS;
}
@@ -911,26 +921,25 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
goto _return;
}
- SCatalog* pCtg = NULL;
+ SCatalog *pCtg = NULL;
code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg);
if (code != TSDB_CODE_SUCCESS) {
goto _return;
}
- char* sql = "taos_load_table_info";
+ char *sql = "taos_load_table_info";
code = buildRequest(pTscObj, sql, strlen(sql), &pRequest);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto _return;
}
-
+
SSyncQueryParam param = {0};
tsem_init(¶m.sem, 0, 0);
param.pRequest = pRequest;
- SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
- .requestId = pRequest->requestId,
- .requestObjRefId = pRequest->self};
+ SRequestConnInfo conn = {
+ .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
@@ -951,7 +960,6 @@ _return:
return code;
}
-
TAOS_STMT *taos_stmt_init(TAOS *taos) {
STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index b6ca3e1350..45b25f9b56 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1170,7 +1170,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
int32_t code = 0;
- ASSERT(numOfRows > 0);
+ // ASSERT(numOfRows > 0);
if (numOfRows == 0) {
return TSDB_CODE_SUCCESS;
@@ -1236,6 +1236,32 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
return 0;
}
+int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
+ ASSERT(src != NULL && dst != NULL);
+
+ blockDataCleanup(dst);
+ int32_t code = blockDataEnsureCapacity(dst, src->info.rows);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ return code;
+ }
+
+ size_t numOfCols = taosArrayGetSize(src->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
+ SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
+ if (pSrc->pData == NULL) {
+ continue;
+ }
+
+ colDataAssign(pDst, pSrc, src->info.rows, &src->info);
+ }
+
+ dst->info.rows = src->info.rows;
+ dst->info.window = src->info.window;
+ return TSDB_CODE_SUCCESS;
+}
+
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
if (pDataBlock == NULL) {
return NULL;
@@ -1613,7 +1639,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
return buf;
}
-void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
+void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
char pBuf[128] = {0};
int32_t sz = taosArrayGetSize(dataBlocks);
for (int32_t i = 0; i < sz; i++) {
@@ -1621,7 +1647,7 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows;
- printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
+ printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
for (int32_t j = 0; j < rows; j++) {
printf("%s |", flag);
for (int32_t k = 0; k < numOfCols; k++) {
@@ -1661,6 +1687,57 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
}
}
+// for debug
+char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
+ int32_t size = 2048;
+ *pDataBuf = taosMemoryCalloc(size, 1);
+ char* dumpBuf = *pDataBuf;
+ char pBuf[128] = {0};
+ int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
+ int32_t rows = pDataBlock->info.rows;
+ int32_t len = 0;
+ len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag,
+ (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
+ for (int32_t j = 0; j < rows; j++) {
+ len += snprintf(dumpBuf + len, size - len, "%s |", flag);
+ for (int32_t k = 0; k < colNum; k++) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+ void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+ if (colDataIsNull(pColInfoData, rows, j, NULL)) {
+ len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
+ continue;
+ }
+ switch (pColInfoData->info.type) {
+ case TSDB_DATA_TYPE_TIMESTAMP:
+ formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
+ len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
+ break;
+ case TSDB_DATA_TYPE_INT:
+ len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var);
+ break;
+ case TSDB_DATA_TYPE_UINT:
+ len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var);
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var);
+ break;
+ case TSDB_DATA_TYPE_UBIGINT:
+ len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var);
+ break;
+ case TSDB_DATA_TYPE_FLOAT:
+ len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var);
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var);
+ break;
+ }
+ }
+ len += snprintf(dumpBuf + len, size - len, "\n");
+ }
+ len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag);
+ return dumpBuf;
+}
+
/**
* @brief TODO: Assume that the final generated result it less than 3M
*
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 8460a27a0e..7c1b31b6e4 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -862,21 +862,6 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) {
printf("\n");
}
-void debugCheckTags(STag *pTag) {
- switch (pTag->flags) {
- case 0x0:
- case 0x20:
- case 0x40:
- case 0x60:
- break;
- default:
- ASSERT(0);
- }
-
- ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0);
- ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver
-}
-
static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
int32_t n = 0;
@@ -999,7 +984,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
debugPrintSTag(*ppTag, __func__, __LINE__);
#endif
- debugCheckTags(*ppTag); // TODO: remove this line after debug
return code;
_err:
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 9db5019512..e9b5c67d76 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -3696,7 +3696,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1;
if (pReq->isTsma) {
- if (tDecodeBinaryAlloc(&decoder, &pReq->pTsma, NULL) < 0) return -1;
+ if (tDecodeBinary(&decoder, (uint8_t **)&pReq->pTsma, NULL) < 0) return -1;
}
tEndDecode(&decoder);
@@ -3707,9 +3707,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
taosArrayDestroy(pReq->pRetensions);
pReq->pRetensions = NULL;
- if (pReq->isTsma) {
- taosMemoryFreeClear(pReq->pTsma);
- }
return 0;
}
@@ -4747,9 +4744,8 @@ int32_t tDecodeSRSmaParam(SDecoder *pCoder, SRSmaParam *pRSmaParam) {
if (tDecodeI64v(pCoder, &pRSmaParam->watermark[i]) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->qmsgLen[i]) < 0) return -1;
if (pRSmaParam->qmsgLen[i] > 0) {
- uint64_t len;
- if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg[i], &len) < 0)
- return -1; // qmsgLen contains len of '\0'
+ tDecoderMalloc(pCoder, pRSmaParam->qmsgLen[i]);
+ if (tDecodeBinary(pCoder, (uint8_t **)&pRSmaParam->qmsg[i], NULL) < 0) return -1; // qmsgLen contains len of '\0'
} else {
pRSmaParam->qmsg[i] = NULL;
}
@@ -4767,7 +4763,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
if (pReq->rollup) {
- if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
+ if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
}
tEndEncode(pCoder);
@@ -4783,7 +4779,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
if (pReq->rollup) {
- if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
+ if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
}
tEndDecode(pCoder);
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index 3d1ece0b64..980749ca70 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -247,11 +247,9 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
return TSDB_CODE_SUCCESS;
}
-
-
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
- int32_t size = 0;
- int32_t rowsRead = 0;
+ int32_t size = 0;
+ int32_t rowsRead = 0;
SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
@@ -259,6 +257,11 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
+ if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
+ terrno = TSDB_CODE_MND_NO_RIGHTS;
+ return -1;
+ }
+
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
@@ -309,7 +312,6 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
-
SArray *dmGetMsgHandles() {
int32_t code = -1;
SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 63d2a65df1..a4745abd5b 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -70,9 +70,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
}
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
- SDnodeTrans *pTrans = &pDnode->trans;
+ SDnodeTrans * pTrans = &pDnode->trans;
int32_t code = -1;
- SRpcMsg *pMsg = NULL;
+ SRpcMsg * pMsg = NULL;
SMgmtWrapper *pWrapper = NULL;
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
@@ -194,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
- SArray *pArray = (*pWrapper->func.getHandlesFp)();
+ SArray * pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
- SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
+ SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
@@ -248,7 +248,14 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
}
}
-static bool rpcRfp(int32_t code) { return code == TSDB_CODE_RPC_REDIRECT; }
+static bool rpcRfp(int32_t code) {
+ if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
+ return true;
+ } else {
+ return false;
+ }
+}
int32_t dmInitClient(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h
index 15f9e4e6b5..0c56f6f155 100644
--- a/source/dnode/mnode/impl/inc/mndPrivilege.h
+++ b/source/dnode/mnode/impl/inc/mndPrivilege.h
@@ -57,6 +57,8 @@ typedef enum {
MND_OPER_USE_DB,
MND_OPER_WRITE_DB,
MND_OPER_READ_DB,
+ MND_OPER_READ_OR_WRITE_DB,
+ MND_OPER_SHOW_VARIBALES,
} EOperType;
int32_t mndInitPrivilege(SMnode *pMnode);
@@ -64,7 +66,8 @@ void mndCleanupPrivilege(SMnode *pMnode);
int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
-int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType);
+int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
+int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
#ifdef __cplusplus
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 69385c3a46..0901e77287 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -33,6 +33,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+// for sma
+// TODO refactor
+int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
#ifdef __cplusplus
}
diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h
index bc2d5c82b1..1497bba11c 100644
--- a/source/dnode/mnode/impl/inc/mndTrans.h
+++ b/source/dnode/mnode/impl/inc/mndTrans.h
@@ -39,8 +39,10 @@ typedef struct {
int32_t id;
int32_t errCode;
int32_t acceptableCode;
- ETrnStage stage;
+ int32_t retryCode;
ETrnAct actionType;
+ ETrnStage stage;
+ int8_t reserved;
int8_t rawWritten;
int8_t msgSent;
int8_t msgReceived;
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 7dc5ee1ea1..5b5de10fba 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -431,6 +431,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
goto SUBSCRIBE_OVER;
}
+ if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) {
+ goto SUBSCRIBE_OVER;
+ }
+
#if 0
// ref topic to prevent drop
// TODO make topic complete
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 0345f1b345..abd1492d5e 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE
#include "mndDb.h"
-#include "mndPrivilege.h"
#include "mndDnode.h"
#include "mndOffset.h"
+#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndSma.h"
#include "mndStb.h"
@@ -1336,7 +1336,7 @@ char *buildRetension(SArray *pRetension) {
}
static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables,
- bool sysDb, ESdbStatus objStatus) {
+ bool sysDb, ESdbStatus objStatus, bool sysinfo) {
int32_t cols = 0;
int32_t bytes = pShow->pMeta->pSchemas[cols].bytes;
@@ -1354,7 +1354,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
char statusB[24] = {0};
STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status));
- if (sysDb) {
+ if (sysDb || !sysinfo) {
for (int32_t i = 0; i < pShow->numOfColumns; ++i) {
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
if (i == 0) {
@@ -1528,17 +1528,21 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
SDbObj *pDb = NULL;
ESdbStatus objStatus = 0;
+ SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
+ if (pUser == NULL) return 0;
+ bool sysinfo = pUser->sysInfo;
+
// Append the information_schema database into the result.
if (!pShow->sysDbRsp) {
SDbObj infoschemaDb = {0};
setInformationSchemaDbCfg(&infoschemaDb);
- dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0);
+ dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0, 1);
numOfRows += 1;
SDbObj perfschemaDb = {0};
setPerfSchemaDbCfg(&perfschemaDb);
- dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0);
+ dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0, 1);
numOfRows += 1;
pShow->sysDbRsp = true;
@@ -1550,16 +1554,19 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
break;
}
- int32_t numOfTables = 0;
- sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
+ if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {
+ int32_t numOfTables = 0;
+ sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
+
+ dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo);
+ numOfRows++;
+ }
- dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus);
- numOfRows++;
sdbRelease(pSdb, pDb);
}
pShow->numOfRows += numOfRows;
-
+ mndReleaseUser(pMnode, pUser);
return numOfRows;
}
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index af1d641ebf..5e78d0f434 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -558,7 +558,11 @@ _OVER:
static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
SShowVariablesRsp rsp = {0};
- int32_t code = -1;
+ int32_t code = -1;
+
+ if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIBALES) != 0) {
+ goto _OVER;
+ }
rsp.variables = taosArrayInit(4, sizeof(SVariablesInfo));
if (NULL == rsp.variables) {
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index 4c9974ba1a..b66082dd98 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -442,7 +442,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
syncPingReplyDestroy(pSyncMsg);
} else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
- code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+ code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
syncClientRequestDestroy(pSyncMsg);
} else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
@@ -491,7 +491,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
syncPingReplyDestroy(pSyncMsg);
} else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
- code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+ code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
syncClientRequestDestroy(pSyncMsg);
} else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c
index 478ba2bee4..dc321e38d1 100644
--- a/source/dnode/mnode/impl/src/mndPrivilege.c
+++ b/source/dnode/mnode/impl/src/mndPrivilege.c
@@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndPrivilege.h"
+#include "mndDb.h"
#include "mndUser.h"
int32_t mndInitPrivilege(SMnode *pMnode) { return 0; }
@@ -45,6 +46,7 @@ int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operTy
case MND_OPER_CONNECT:
case MND_OPER_CREATE_FUNC:
case MND_OPER_DROP_FUNC:
+ case MND_OPER_SHOW_VARIBALES:
break;
default:
terrno = TSDB_CODE_MND_NO_RIGHTS;
@@ -79,7 +81,7 @@ int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterU
return -1;
}
-int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) {
+int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname) {
int32_t code = 0;
SUserObj *pUser = mndAcquireUser(pMnode, user);
@@ -98,14 +100,34 @@ int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType
goto _OVER;
}
- if (!pUser->sysInfo) {
- terrno = TSDB_CODE_MND_NO_RIGHTS;
- code = -1;
+ if (pUser->sysInfo) {
goto _OVER;
}
- terrno = TSDB_CODE_MND_NO_RIGHTS;
- code = -1;
+ switch (showType) {
+ case TSDB_MGMT_TABLE_DB:
+ case TSDB_MGMT_TABLE_STB:
+ case TSDB_MGMT_TABLE_INDEX:
+ case TSDB_MGMT_TABLE_STREAMS:
+ case TSDB_MGMT_TABLE_CONSUMERS:
+ case TSDB_MGMT_TABLE_TOPICS:
+ case TSDB_MGMT_TABLE_SUBSCRIPTIONS:
+ case TSDB_MGMT_TABLE_FUNC:
+ case TSDB_MGMT_TABLE_QUERIES:
+ case TSDB_MGMT_TABLE_CONNS:
+ case TSDB_MGMT_TABLE_APPS:
+ case TSDB_MGMT_TABLE_TRANS:
+ code = 0;
+ break;
+ default:
+ terrno = TSDB_CODE_MND_NO_RIGHTS;
+ code = -1;
+ goto _OVER;
+ }
+
+ if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP || showType == TSDB_MGMT_TABLE_INDEX) {
+ code = mndCheckDbPrivilegeByName(pMnode, user, MND_OPER_READ_OR_WRITE_DB, dbname);
+ }
_OVER:
mndReleaseUser(pMnode, pUser);
@@ -133,19 +155,11 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
if (pUser->sysInfo) goto _OVER;
}
- if (operType == MND_OPER_ALTER_DB) {
+ if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) {
if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
}
- if (operType == MND_OPER_DROP_DB) {
- if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
- }
-
- if (operType == MND_OPER_COMPACT_DB) {
- if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
- }
-
- if (operType == MND_OPER_USE_DB) {
+ if (operType == MND_OPER_USE_DB || operType == MND_OPER_READ_OR_WRITE_DB) {
if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER;
if (taosHashGet(pUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER;
if (taosHashGet(pUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER;
@@ -168,3 +182,12 @@ _OVER:
mndReleaseUser(pMnode, pUser);
return code;
}
+
+int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
+ SDbObj *pDb = mndAcquireDb(pMnode, dbname);
+ if (pDb == NULL) return -1;
+
+ int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb);
+ mndReleaseDb(pMnode, pDb);
+ return code;
+}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c
index f2e599b073..38368b4ece 100644
--- a/source/dnode/mnode/impl/src/mndProfile.c
+++ b/source/dnode/mnode/impl/src/mndProfile.c
@@ -254,6 +254,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
terrstr());
goto _OVER;
}
+
+ if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) {
+ goto _OVER;
+ }
}
pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp,
diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c
index 3351cff3a3..164bcc7d60 100644
--- a/source/dnode/mnode/impl/src/mndShow.c
+++ b/source/dnode/mnode/impl/src/mndShow.c
@@ -231,8 +231,14 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
}
mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
-
- // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1;
+ if (retrieveReq.user[0] != 0) {
+ memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN);
+ } else {
+ memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
+ }
+ if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
+ return -1;
+ }
int32_t numOfCols = pShow->pMeta->numOfColumns;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index ef24cd0ba4..10eeaba982 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -15,11 +15,11 @@
#define _DEFAULT_SOURCE
#include "mndSma.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
+#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndStb.h"
@@ -857,6 +857,23 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name);
mndTransSetDbName(pTrans, pDb->name, NULL);
+ SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name);
+ if (pStream == NULL || pStream->smaId != pSma->uid) {
+ sdbRelease(pMnode->pSdb, pStream);
+ goto _OVER;
+ } else {
+ if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+ mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
+ sdbRelease(pMnode->pSdb, pStream);
+ goto _OVER;
+ }
+
+ // drop stream
+ if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
+ sdbRelease(pMnode->pSdb, pStream);
+ goto _OVER;
+ }
+ }
if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 77b13cd82d..dd01a0fa16 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -15,12 +15,12 @@
#define _DEFAULT_SOURCE
#include "mndStb.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
#include "mndPerfSchema.h"
+#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndSma.h"
@@ -107,7 +107,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
int32_t funcNum = taosArrayGetSize(pStb->pFuncs);
SDB_SET_INT32(pRaw, dataPos, funcNum, _OVER)
for (int32_t i = 0; i < funcNum; ++i) {
- char* func = taosArrayGet(pStb->pFuncs, i);
+ char *func = taosArrayGet(pStb->pFuncs, i);
SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER)
}
@@ -427,17 +427,17 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
req.schemaTag.pSchema = pStb->pTags;
if (req.rollup) {
- req.pRSmaParam.maxdelay[0] = pStb->maxdelay[0];
- req.pRSmaParam.maxdelay[1] = pStb->maxdelay[1];
+ req.rsmaParam.maxdelay[0] = pStb->maxdelay[0];
+ req.rsmaParam.maxdelay[1] = pStb->maxdelay[1];
if (pStb->ast1Len > 0) {
- if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[0], &req.pRSmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
- STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[0]) < 0) {
+ if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
+ STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) {
goto _err;
}
}
if (pStb->ast2Len > 0) {
- if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[1], &req.pRSmaParam.qmsgLen[1], pStb->pAst2, pStb->uid,
- STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[1]) < 0) {
+ if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid,
+ STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) {
goto _err;
}
}
@@ -470,12 +470,12 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
tEncoderClear(&encoder);
*pContLen = contLen;
- taosMemoryFreeClear(req.pRSmaParam.qmsg[0]);
- taosMemoryFreeClear(req.pRSmaParam.qmsg[1]);
+ taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
+ taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
return pHead;
_err:
- taosMemoryFreeClear(req.pRSmaParam.qmsg[0]);
- taosMemoryFreeClear(req.pRSmaParam.qmsg[1]);
+ taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
+ taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
return NULL;
}
@@ -708,7 +708,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pDst->commentLen = pCreate->commentLen;
pDst->pFuncs = pCreate->pFuncs;
pCreate->pFuncs = NULL;
-
+
if (pDst->commentLen > 0) {
pDst->comment = taosMemoryCalloc(pDst->commentLen + 1, 1);
if (pDst->comment == NULL) {
@@ -1391,12 +1391,11 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
if (pStb->pFuncs) {
pRsp->pFuncs = taosArrayDup(pStb->pFuncs);
}
-
+
taosRUnLockLatch(&pStb->lock);
return 0;
}
-
static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp,
int32_t *smaVer) {
char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
@@ -1426,31 +1425,29 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char
}
static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
- char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
- snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
+ char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
+ snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
- SDbObj *pDb = mndAcquireDb(pMnode, dbFName);
- if (pDb == NULL) {
- terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
- return -1;
- }
+ SDbObj *pDb = mndAcquireDb(pMnode, dbFName);
+ if (pDb == NULL) {
+ terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+ return -1;
+ }
- SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
- if (pStb == NULL) {
- mndReleaseDb(pMnode, pDb);
- terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
- return -1;
- }
+ SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
+ if (pStb == NULL) {
+ mndReleaseDb(pMnode, pDb);
+ terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+ return -1;
+ }
- int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp);
-
- mndReleaseDb(pMnode, pDb);
- mndReleaseStb(pMnode, pStb);
- return code;
+ int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp);
+
+ mndReleaseDb(pMnode, pDb);
+ mndReleaseStb(pMnode, pStb);
+ return code;
}
-
-
static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont,
int32_t *pLen) {
int32_t ret;
@@ -1811,10 +1808,10 @@ _OVER:
}
static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) {
- SMnode *pMnode = pReq->info.node;
- int32_t code = -1;
- STableCfgReq cfgReq = {0};
- STableCfgRsp cfgRsp = {0};
+ SMnode *pMnode = pReq->info.node;
+ int32_t code = -1;
+ STableCfgReq cfgReq = {0};
+ STableCfgRsp cfgRsp = {0};
if (tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
@@ -1866,7 +1863,6 @@ _OVER:
return code;
}
-
int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp,
int32_t *pRspLen) {
SSTbHbRsp hbRsp = {0};
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index b865d0495c..cabbac14f1 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -14,10 +14,10 @@
*/
#include "mndStream.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
+#include "mndPrivilege.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndStb.h"
@@ -437,10 +437,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
goto _OVER;
}
- if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) {
- goto _OVER;
- }
-
int32_t numOfStbs = -1;
if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) {
goto _OVER;
@@ -494,7 +490,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
return 0;
}
-static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
+int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
int32_t lv = taosArrayGetSize(pStream->tasks);
for (int32_t i = 0; i < lv; i++) {
SArray *pTasks = taosArrayGetP(pStream->tasks, i);
@@ -542,19 +538,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
- // TODO check read auth for source and write auth for target
-#if 0
- pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB);
- if (pDb == NULL) {
- terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
- goto _OVER;
- }
-
- if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
- goto _OVER;
- }
-#endif
-
// build stream obj from request
SStreamObj streamObj = {0};
if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) {
@@ -592,6 +575,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) {
+ mndTransDrop(pTrans);
+ goto _OVER;
+ }
+
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) {
+ mndTransDrop(pTrans);
+ goto _OVER;
+ }
+
// execute creation
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -641,13 +634,9 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
}
}
-#if 0
- // todo check auth
- pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
- if (pUser == NULL) {
- goto DROP_STREAM_OVER;
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
+ return -1;
}
-#endif
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
if (pTrans == NULL) {
diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c
index b8c17378c4..8d1220d596 100644
--- a/source/dnode/mnode/impl/src/mndTopic.c
+++ b/source/dnode/mnode/impl/src/mndTopic.c
@@ -14,12 +14,12 @@
*/
#include "mndTopic.h"
-#include "mndPrivilege.h"
#include "mndConsumer.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
+#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndSubscribe.h"
@@ -401,6 +401,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
}
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName);
+ if (pStb == NULL) {
+ terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+ return -1;
+ }
topicObj.stbUid = pStb->uid;
}
/*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/
@@ -480,7 +484,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
goto _OVER;
}
- if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+ if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) {
goto _OVER;
}
@@ -571,6 +575,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
}
#endif
+ if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) {
+ return -1;
+ }
+
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq);
mndTransSetDbName(pTrans, pTopic->db, NULL);
if (pTrans == NULL) {
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index a9de1a05a7..b09ee5f608 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -15,15 +15,15 @@
#define _DEFAULT_SOURCE
#include "mndTrans.h"
-#include "mndPrivilege.h"
#include "mndConsumer.h"
#include "mndDb.h"
+#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndSync.h"
#include "mndUser.h"
-#define TRANS_VER_NUMBER 1
-#define TRANS_ARRAY_SIZE 8
+#define TRANS_VER_NUMBER 1
+#define TRANS_ARRAY_SIZE 8
#define TRANS_RESERVE_SIZE 64
static SSdbRaw *mndTransActionEncode(STrans *pTrans);
@@ -55,7 +55,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
-static int32_t mndProcessTransReq(SRpcMsg *pReq);
+static int32_t mndProcessTransTimer(SRpcMsg *pReq);
static int32_t mndProcessTtl(SRpcMsg *pReq);
static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
@@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
.deleteFp = (SdbDeleteFp)mndTransActionDelete,
};
- mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq);
+ mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer);
mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans);
@@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
+ SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
+ SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
if (pAction->actionType == TRANS_ACTION_RAW) {
int32_t len = sdbGetRawTotalSize(pAction->pRaw);
SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
+ SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
+ SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
if (pAction->actionType == TRANS_ACTION_RAW) {
int32_t len = sdbGetRawTotalSize(pAction->pRaw);
SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
+ SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
+ SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
if (pAction->actionType == TRANS_ACTION_RAW) {
int32_t len = sdbGetRawTotalSize(pAction->pRaw);
SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
+ SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
action.actionType = actionType;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
action.stage = stage;
+ SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
if (action.actionType == TRANS_ACTION_RAW) {
SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
+ SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
action.actionType = actionType;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
action.stage = stage;
+ SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
if (action.actionType == TRANS_ACTION_RAW) {
SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
+ SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
action.actionType = actionType;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
action.stage = stage;
+ SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
if (action.actionType) {
SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
+ pTrans->startFunc = 0;
}
return 0;
}
static void mndTransDropData(STrans *pTrans) {
- mndTransDropActions(pTrans->redoActions);
- mndTransDropActions(pTrans->undoActions);
- mndTransDropActions(pTrans->commitActions);
+ if (pTrans->redoActions != NULL) {
+ mndTransDropActions(pTrans->redoActions);
+ pTrans->redoActions = NULL;
+ }
+ if (pTrans->undoActions != NULL) {
+ mndTransDropActions(pTrans->undoActions);
+ pTrans->undoActions = NULL;
+ }
+ if (pTrans->commitActions != NULL) {
+ mndTransDropActions(pTrans->commitActions);
+ pTrans->commitActions = NULL;
+ }
if (pTrans->rpcRsp != NULL) {
taosMemoryFree(pTrans->rpcRsp);
pTrans->rpcRsp = NULL;
@@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
+ pTrans->stopFunc = 0;
}
mndTransDropData(pTrans);
@@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
sendRsp = true;
}
} else {
- if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) {
+ if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) {
if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
sendRsp = true;
}
@@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
pAction->errCode = pRsp->code;
}
- mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action,
- pRsp->code, pAction->acceptableCode);
+ mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId,
+ mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
mndTransExecute(pMnode, pTrans);
_OVER:
@@ -884,6 +907,21 @@ _OVER:
return 0;
}
+static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
+ pAction->rawWritten = 0;
+ pAction->msgSent = 0;
+ pAction->msgReceived = 0;
+ if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
+ pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
+ pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
+ mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
+ pAction->id, pAction->epSet.inUse);
+ } else {
+ mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id);
+ }
+ pAction->errCode = 0;
+}
+
static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) {
int32_t numOfActions = taosArrayGetSize(pArray);
@@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
continue;
if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue;
- pAction->rawWritten = 0;
- pAction->msgSent = 0;
- pAction->msgReceived = 0;
- if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
- pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
- pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
- mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
- action, pAction->epSet.inUse);
- } else {
- mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action);
- }
- pAction->errCode = 0;
+ mndTransResetAction(pMnode, pTrans, pAction);
}
}
@@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
if (pAction->msgReceived) {
if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
code = pAction->errCode;
- pAction->msgSent = 0;
- pAction->msgReceived = 0;
- mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action);
+ mndTransResetAction(pMnode, pTrans, pAction);
+ } else {
+ mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action);
}
} else {
code = TSDB_CODE_ACTION_IN_PROGRESS;
@@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
if (pAction->rawWritten) {
if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
code = pAction->errCode;
+ } else {
+ mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action);
}
}
}
@@ -1156,9 +1185,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
} else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id);
break;
+ } else if (code == pAction->retryCode) {
+ mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code);
+ taosMsleep(300);
+ action--;
+ continue;
} else {
terrno = code;
pTrans->code = code;
+ mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id,
+ mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes);
break;
}
}
@@ -1343,7 +1379,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
mndTransSendRpcRsp(pMnode, pTrans);
}
-static int32_t mndProcessTransReq(SRpcMsg *pReq) {
+static int32_t mndProcessTransTimer(SRpcMsg *pReq) {
mndTransPullup(pReq->info.node);
return 0;
}
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index 0e931e0a9c..0e3271a1bb 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -15,10 +15,10 @@
#define _DEFAULT_SOURCE
#include "mndVgroup.h"
-#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
+#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
@@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD
action.pCont = pHead;
action.contLen = contLen;
action.msgType = TDMT_VND_ALTER_CONFIRM;
+ // incorrect redirect result will cause this erro
+ action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pHead);
@@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb
action.contLen = contLen;
action.msgType = TDMT_SYNC_SET_VNODE_STANDBY;
action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ // Keep retrying until the target vnode is not the leader
+ action.retryCode = TSDB_CODE_SYN_IS_LEADER;
if (isRedo) {
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -1229,7 +1233,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
}
if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) {
- terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+ // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+ code = 0;
goto _OVER;
}
@@ -1351,7 +1356,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
}
if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) {
- terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+ // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+ code = 0;
goto _OVER;
}
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 3a92cba773..b13e654caf 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -257,7 +257,7 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
case TDMT_STREAM_TASK_RECOVER_RSP:
return sndProcessTaskRecoverRsp(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE_RSP:
- return sndProcessTaskRecoverRsp(pSnode, pMsg);
+ return sndProcessTaskRetrieveRsp(pSnode, pMsg);
default:
ASSERT(0);
}
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index a32bf0ecdb..5c2d2cd712 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -116,18 +116,19 @@ typedef void *tsdbReaderT;
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3
-int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList);
-tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
- uint64_t taskId);
-tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
- void *pMemRef);
-int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
-bool isTsdbCacheLastRow(tsdbReaderT *pReader);
-int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
-int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
-void *tsdbGetIdx(SMeta *pMeta);
-void *tsdbGetIvtIdx(SMeta *pMeta);
-int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
+int32_t tsdbSetTableList(tsdbReaderT reader, SArray *tableList);
+tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
+ uint64_t taskId);
+tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
+ void *pMemRef);
+int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
+bool isTsdbCacheLastRow(tsdbReaderT *pReader);
+int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
+int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
+int32_t tsdbGetStbIdList(SMeta *pMeta, int64_t suid, SArray *list);
+void *tsdbGetIdx(SMeta *pMeta);
+void *tsdbGetIvtIdx(SMeta *pMeta);
+int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
@@ -150,8 +151,7 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle);
bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
- int32_t *pNumOfRows);
+int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@@ -196,17 +196,23 @@ struct SVnodeCfg {
typedef struct {
TSKEY lastKey;
uint64_t uid;
+ uint64_t groupId;
} STableKeyInfo;
+#define TABLE_ROLLUP_ON ((int8_t)0x1)
+#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0)
+#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
struct SMetaEntry {
int64_t version;
int8_t type;
+ int8_t flags; // TODO: need refactor?
tb_uid_t uid;
char *name;
union {
struct {
SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag;
+ SRSmaParam rsmaParam;
} stbEntry;
struct {
int64_t ctime;
diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index 63a8e7ec68..66d1689d57 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -69,6 +69,7 @@ struct SMeta {
TTB* pUidIdx;
TTB* pNameIdx;
TTB* pCtbIdx;
+ TTB* pSuidIdx;
// ivt idx and idx
void* pTagIvtIdx;
TTB* pTagIdx;
diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h
index 1489b02492..956e451b58 100644
--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@@ -60,8 +60,9 @@ struct SRSmaStat {
SSma *pSma;
void *tmrHandle;
tmr_h tmrId;
- int8_t tmrStat;
int32_t tmrSeconds;
+ int8_t triggerStat;
+ int8_t runningStat;
SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo;
};
@@ -72,13 +73,21 @@ struct SSmaStat {
};
T_REF_DECLARE()
};
-#define SMA_TSMA_STAT(s) (&(s)->tsmaStat)
-#define SMA_RSMA_STAT(s) (&(s)->rsmaStat)
-#define SMA_RSMA_INFO_HASH(s) ((s)->rsmaStat.rsmaInfoHash)
-#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle)
-#define SMA_RSMA_TMR_STAT(s) ((s)->rsmaStat.tmrStat)
-#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash)
+#define SMA_TSMA_STAT(s) (&(s)->tsmaStat)
+#define SMA_RSMA_STAT(s) (&(s)->rsmaStat)
+#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash)
+#define RSMA_TMR_ID(r) ((r)->tmrId)
+#define RSMA_TMR_HANDLE(r) ((r)->tmrHandle)
+#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
+#define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
+enum {
+ TASK_TRIGGER_STAT_INIT = 0,
+ TASK_TRIGGER_STAT_ACTIVE = 1,
+ TASK_TRIGGER_STAT_INACTIVE = 2,
+ TASK_TRIGGER_STAT_CANCELLED = 3,
+ TASK_TRIGGER_STAT_FINISHED = 4,
+};
void tdDestroySmaEnv(SSmaEnv *pSmaEnv);
void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
@@ -174,6 +183,8 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
+// smaFileUtil ================
+
typedef struct STFInfo STFInfo;
typedef struct STFile STFile;
@@ -181,7 +192,7 @@ struct STFInfo {
uint32_t magic;
uint32_t ftype;
uint32_t fver;
- uint64_t fsize;
+ int64_t fsize;
};
struct STFile {
@@ -199,11 +210,8 @@ struct STFile {
#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL)
#define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf))
#define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL)
-#define TD_FILE_STATE(tf) ((tf)->state)
#define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TD_FILE_DID(tf) (TD_FILE_F(tf)->did)
-#define TD_FILE_IS_OK(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_OK)
-#define TD_FILE_IS_BAD(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD)
int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 3b30224e17..68ed6dde51 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -76,6 +76,7 @@ void vnodeFree(void* p);
// meta
typedef struct SMCtbCursor SMCtbCursor;
+typedef struct SMStbCursor SMStbCursor;
typedef struct STbUidStore STbUidStore;
int metaOpen(SVnode* pVnode, SMeta** ppMeta);
@@ -97,6 +98,9 @@ int metaGetTbNum(SMeta* pMeta);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseCtbCursor(SMCtbCursor* pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
+SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid);
+void metaCloseStbCursor(SMStbCursor* pStbCur);
+tb_uid_t metaStbCursorNext(SMStbCursor* pStbCur);
STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
@@ -158,6 +162,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
// sma
int32_t smaOpen(SVnode* pVnode);
int32_t smaClose(SSma* pSma);
+int32_t smaCloseEnv(SSma* pSma);
+int32_t smaCloseEx(SSma* pSma);
int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c
index acf5b0b613..23d7665ba3 100644
--- a/source/dnode/vnode/src/meta/metaEntry.c
+++ b/source/dnode/vnode/src/meta/metaEntry.c
@@ -24,22 +24,25 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tEncodeCStr(pCoder, pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) {
+ if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor?
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+ if (TABLE_IS_ROLLUP(pME->flags)) {
+ if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
+ }
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
- if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
+ if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
if (pME->ctbEntry.commentLen > 0){
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1;
}
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
- debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
- if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
+ if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
if (pME->ntbEntry.commentLen > 0){
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1;
}
@@ -64,23 +67,26 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) {
+ if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor?
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+ if (TABLE_IS_ROLLUP(pME->flags)) {
+ if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
+ }
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
- if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
+ if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
if (pME->ctbEntry.commentLen > 0){
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0)
return -1;
}
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO)
- debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
- if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
+ if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
if (pME->ntbEntry.commentLen > 0){
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1;
}
diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c
index 95510c25e5..614ccc5b5e 100644
--- a/source/dnode/vnode/src/meta/metaOpen.c
+++ b/source/dnode/vnode/src/meta/metaOpen.c
@@ -92,6 +92,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
goto _err;
}
+ // open pSuidIdx
+ ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx);
+ if (ret < 0) {
+ metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
+
// open pTagIdx
// TODO(yihaoDeng), refactor later
char indexFullPath[128] = {0};
@@ -141,6 +148,7 @@ _err:
if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
+ if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx);
if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
@@ -162,6 +170,7 @@ int metaClose(SMeta *pMeta) {
if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
#endif
if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
+ if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx);
if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 910f4bba51..85106f46c2 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -325,6 +325,72 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
return pCtbIdxKey->uid;
}
+struct SMStbCursor {
+ SMeta *pMeta;
+ TBC *pCur;
+ tb_uid_t suid;
+ void *pKey;
+ void *pVal;
+ int kLen;
+ int vLen;
+};
+
+SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) {
+ SMStbCursor *pStbCur = NULL;
+ int ret = 0;
+ int c = 0;
+
+ pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur));
+ if (pStbCur == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ pStbCur->pMeta = pMeta;
+ pStbCur->suid = suid;
+ metaRLock(pMeta);
+
+ ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL);
+ if (ret < 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ metaULock(pMeta);
+ taosMemoryFree(pStbCur);
+ return NULL;
+ }
+
+ // move to the suid
+ tdbTbcMoveTo(pStbCur->pCur, &suid, sizeof(suid), &c);
+ if (c > 0) {
+ tdbTbcMoveToNext(pStbCur->pCur);
+ }
+
+ return pStbCur;
+}
+
+void metaCloseStbCursor(SMStbCursor *pStbCur) {
+ if (pStbCur) {
+ if (pStbCur->pMeta) metaULock(pStbCur->pMeta);
+ if (pStbCur->pCur) {
+ tdbTbcClose(pStbCur->pCur);
+
+ tdbFree(pStbCur->pKey);
+ tdbFree(pStbCur->pVal);
+ }
+
+ taosMemoryFree(pStbCur);
+ }
+}
+
+tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) {
+ int ret;
+
+ ret = tdbTbcNext(pStbCur->pCur, &pStbCur->pKey, &pStbCur->kLen, &pStbCur->pVal, &pStbCur->vLen);
+ if (ret < 0) {
+ return 0;
+ }
+ return *(tb_uid_t*)pStbCur->pKey;
+}
+
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
// SMetaReader mr = {0};
STSchema * pTSchema = NULL;
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 4a3feca8d0..a621b4ddb0 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -23,6 +23,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
@@ -138,6 +139,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
me.name = pReq->name;
me.stbEntry.schemaRow = pReq->schemaRow;
me.stbEntry.schemaTag = pReq->schemaTag;
+ if (pReq->rollup) {
+ TABLE_SET_ROLLUP(me.flags);
+ me.stbEntry.rsmaParam = pReq->rsmaParam;
+ }
if (metaHandleEntry(pMeta, &me) < 0) goto _err;
@@ -209,6 +214,7 @@ _drop_super_table:
&pMeta->txn);
tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn);
tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn);
+ tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn);
metaULock(pMeta);
@@ -388,7 +394,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
}
static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
- int32_t ttlDays;
+ int64_t ttlDays;
int64_t ctime;
if (pME->type == TSDB_CHILD_TABLE) {
ctime = pME->ctbEntry.ctime;
@@ -436,11 +442,13 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
+
if (e.type == TSDB_CHILD_TABLE) {
tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn);
} else if (e.type == TSDB_NORMAL_TABLE) {
// drop schema.db (todo)
} else if (e.type == TSDB_SUPER_TABLE) {
+ tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn);
// drop schema.db (todo)
}
@@ -911,6 +919,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
}
+static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) {
+ return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn);
+}
+
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
}
@@ -1081,6 +1093,10 @@ static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
} else {
// update schema.db
if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err;
+
+ if (pME->type == TSDB_SUPER_TABLE) {
+ if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err;
+ }
}
if (pME->type != TSDB_SUPER_TABLE) {
diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c
index 4e907e93ca..7f115633b9 100644
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@@ -126,22 +126,21 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
}
if (smaType == TSDB_SMA_TYPE_ROLLUP) {
- SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma;
+ SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
+ pRSmaStat->pSma = (SSma *)pSma;
// init timer
- SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G");
- if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) {
+ RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
+ if (!RSMA_TMR_HANDLE(pRSmaStat)) {
taosMemoryFreeClear(*pSmaStat);
return TSDB_CODE_FAILED;
}
-
- atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE);
// init hash
- SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit(
+ RSMA_INFO_HASH(pRSmaStat) = taosHashInit(
RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
- if (!SMA_RSMA_INFO_HASH(*pSmaStat)) {
- if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) {
- taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat));
+ if (!RSMA_INFO_HASH(pRSmaStat)) {
+ if (RSMA_TMR_HANDLE(pRSmaStat)) {
+ taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat));
}
taosMemoryFreeClear(*pSmaStat);
return TSDB_CODE_FAILED;
@@ -155,12 +154,80 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
return TSDB_CODE_SUCCESS;
}
-static void *tdFreeTSmaStat(STSmaStat *pStat) {
+static void tdDestroyTSmaStat(STSmaStat *pStat) {
if (pStat) {
+ smaDebug("destroy tsma stat");
tDestroyTSma(pStat->pTSma);
taosMemoryFreeClear(pStat->pTSma);
- taosMemoryFreeClear(pStat);
+ taosMemoryFreeClear(pStat->pTSchema);
}
+}
+
+static void *tdFreeTSmaStat(STSmaStat *pStat) {
+ tdDestroyTSmaStat(pStat);
+ taosMemoryFreeClear(pStat);
+ return NULL;
+}
+
+static void tdDestroyRSmaStat(SRSmaStat *pStat) {
+ if (pStat) {
+ smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma));
+ // step 1: set persistence task cancelled
+ atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
+
+ // step 2: stop the persistence timer
+ taosTmrStopA(&RSMA_TMR_ID(pStat));
+
+ // step 3: wait the persistence thread to finish
+ int32_t nLoops = 0;
+ if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) {
+ while (1) {
+ if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) {
+ break;
+ } else {
+ smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
+ }
+ ++nLoops;
+ if (nLoops > 1000) {
+ sched_yield();
+ nLoops = 0;
+ }
+ }
+ }
+
+ // step 4: destroy the rsma info and associated fetch tasks
+ // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
+ void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
+ while (infoHash) {
+ SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
+ tdFreeRSmaInfo(pSmaInfo);
+ infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
+ }
+ taosHashCleanup(RSMA_INFO_HASH(pStat));
+
+ // step 5: wait all triggered fetch tasks finished
+ nLoops = 0;
+ while (1) {
+ if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
+ break;
+ }
+ ++nLoops;
+ if (nLoops > 1000) {
+ sched_yield();
+ nLoops = 0;
+ }
+ }
+
+ // step 6: cleanup the timer handle
+ if (RSMA_TMR_HANDLE(pStat)) {
+ taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
+ }
+ }
+}
+
+static void *tdFreeRSmaStat(SRSmaStat *pStat) {
+ tdDestroyRSmaStat(pStat);
+ taosMemoryFreeClear(pStat);
return NULL;
}
@@ -179,19 +246,9 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
if (pSmaStat) {
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
- tdFreeTSmaStat(&pSmaStat->tsmaStat);
+ tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
} else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
- if (SMA_RSMA_TMR_HANDLE(pSmaStat)) {
- taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat));
- }
- // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
- void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL);
- while (infoHash) {
- SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash;
- tdFreeRSmaInfo(pInfoHash);
- infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash);
- }
- taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat));
+ tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat));
} else {
ASSERT(0);
}
@@ -259,35 +316,4 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
tdUnLockSma(pSma);
return TSDB_CODE_SUCCESS;
-};
-
-int32_t smaTimerInit(void **timer, int8_t *initFlag, const char *label) {
- int8_t old;
- while (1) {
- old = atomic_val_compare_exchange_8(initFlag, 0, 2);
- if (old != 2) break;
- }
-
- if (old == 0) {
- *timer = taosTmrInit(10000, 100, 10000, label);
- if (!(*timer)) {
- atomic_store_8(initFlag, 0);
- return -1;
- }
- atomic_store_8(initFlag, 1);
- }
- return 0;
-}
-
-void smaTimerCleanUp(void *timer, int8_t *initFlag) {
- int8_t old;
- while (1) {
- old = atomic_val_compare_exchange_8(initFlag, 1, 2);
- if (old != 2) break;
- }
-
- if (old == 1) {
- taosTmrCleanUp(timer);
- atomic_store_8(initFlag, 0);
- }
-}
\ No newline at end of file
+};
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c
index 383c264e7b..fad31f92c0 100644
--- a/source/dnode/vnode/src/sma/smaOpen.c
+++ b/source/dnode/vnode/src/sma/smaOpen.c
@@ -18,6 +18,7 @@
static int32_t smaEvalDays(SRetention *r, int8_t precision);
static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
+static int32_t rsmaRestore(SSma *pSma);
#define SMA_SET_KEEP_CFG(l) \
do { \
@@ -100,6 +101,9 @@ int32_t smaOpen(SVnode *pVnode) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
+
+ pVnode->pSma = pSma;
+
pSma->pVnode = pVnode;
taosThreadMutexInit(&pSma->mutex, NULL);
pSma->locked = false;
@@ -117,37 +121,83 @@ int32_t smaOpen(SVnode *pVnode) {
ASSERT(0);
}
}
+
+ // restore the rsma
+ if (rsmaRestore(pSma) < 0) {
+ goto _err;
+ }
}
- pVnode->pSma = pSma;
return 0;
_err:
taosMemoryFreeClear(pSma);
return -1;
}
-int32_t smaClose(SSma *pSma) {
+int32_t smaCloseEnv(SSma *pSma) {
+ if (pSma) {
+ SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
+ SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
+ }
+ return 0;
+}
+
+int32_t smaCloseEx(SSma *pSma) {
if (pSma) {
taosThreadMutexDestroy(&pSma->mutex);
if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));
- // SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
- // SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
taosMemoryFreeClear(pSma);
}
return 0;
}
+int32_t smaClose(SSma *pSma) {
+ smaCloseEnv(pSma);
+ smaCloseEx(pSma);
+ return 0;
+}
+
/**
* @brief rsma env restore
- *
- * @param pSma
- * @return int32_t
+ *
+ * @param pSma
+ * @return int32_t
*/
-int32_t smaRestore(SSma *pSma) {
- if (!pSma) return 0;
+static int32_t rsmaRestore(SSma *pSma) {
+ ASSERT(VND_IS_RSMA(pSma->pVnode));
+
// iterate all stables to restore the rsma env
-
+ SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
+ if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
+ smaError("failed to restore rsma since get stb id list error: %s", terrstr());
+ return TSDB_CODE_FAILED;
+ }
+
+ SMetaReader mr = {0};
+ metaReaderInit(&mr, SMA_META(pSma), 0);
+ for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) {
+ tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
+ smaDebug("suid [%d] is %" PRIi64, i, suid);
+ if (metaGetTableEntryByUid(&mr, suid) < 0) {
+ metaReaderClear(&mr);
+ taosArrayDestroy(suidList);
+ smaError("failed to get table meta for %" PRIi64 " since %s", suid, terrstr());
+ return TSDB_CODE_FAILED;
+ }
+ ASSERT(mr.me.type == TSDB_SUPER_TABLE);
+ if (TABLE_IS_ROLLUP(mr.me.flags)) {
+ SRSmaParam *param = &mr.me.stbEntry.rsmaParam;
+ for (int i = 0; i < 2; ++i) {
+ smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode),
+ suid, i, param->maxdelay[i], i, param->watermark[i]);
+ }
+ }
+ }
+
+ metaReaderClear(&mr);
+ taosArrayDestroy(suidList);
+
return TSDB_CODE_SUCCESS;
}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 3c2f710fa7..4ffd52f189 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -14,8 +14,8 @@
*/
#include "sma.h"
-#include "tstream.h"
+#define RSMA_QTASK_PERSIST_MS 7200000
typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
@@ -31,13 +31,13 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId);
struct SRSmaInfoItem {
SRSmaInfo *pRsmaInfo;
void *taskInfo; // qTaskInfo_t
- void *tmrHandle;
tmr_h tmrId;
int8_t level;
int8_t tmrInitFlag;
- int8_t triggerStatus; // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE
+ int8_t triggerStat;
int32_t maxDelay;
};
+
struct SRSmaInfo {
STSchema *pTSchema;
SSma *pSma;
@@ -45,11 +45,14 @@ struct SRSmaInfo {
SRSmaInfoItem items[TSDB_RETENTION_L2];
};
-static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) {
+static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) {
// Note: free/kill may in RC
qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
+ smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level);
qDestroyTask(otaskHandle);
+ } else {
+ smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level);
}
}
@@ -58,14 +61,19 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
SRSmaInfoItem *pItem = &pInfo->items[i];
if (pItem->taskInfo) {
- tdFreeTaskHandle(pItem->taskInfo);
- }
- if (pItem->tmrHandle) {
- taosTmrCleanUp(pItem->tmrHandle);
+ smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid,
+ pItem->tmrId, i + 1);
+ taosTmrStopA(&pItem->tmrId);
+ tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1);
+ } else {
+ smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo",
+ SMA_VID(pInfo->pSma), pInfo->suid, i + 1);
}
}
taosMemoryFree(pInfo->pTSchema);
taosMemoryFree(pInfo);
+ } else {
+ smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid);
}
return NULL;
@@ -81,9 +89,9 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
return TSDB_CODE_SUCCESS;
}
-static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
+static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
- SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
if (!suid || !tbUids) {
@@ -92,27 +100,31 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA
return TSDB_CODE_FAILED;
}
- pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
+ pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
terrno = TSDB_CODE_RSMA_INVALID_STAT;
return TSDB_CODE_FAILED;
}
- if (pRSmaInfo->items[0].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) {
- smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
- return TSDB_CODE_FAILED;
- } else {
- smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
- pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+ if (pRSmaInfo->items[0].taskInfo) {
+ if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) {
+ smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+ return TSDB_CODE_FAILED;
+ } else {
+ smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
+ pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+ }
}
- if (pRSmaInfo->items[1].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) {
- smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
- return TSDB_CODE_FAILED;
- } else {
- smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
- pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+ if (pRSmaInfo->items[1].taskInfo) {
+ if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) {
+ smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+ return TSDB_CODE_FAILED;
+ } else {
+ smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
+ pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+ }
}
return TSDB_CODE_SUCCESS;
@@ -159,9 +171,9 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
return TSDB_CODE_SUCCESS;
}
- SSmaStat *pStat = SMA_ENV_STAT(pEnv);
- SHashObj *infoHash = NULL;
- if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) {
+ SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+ SHashObj *infoHash = NULL;
+ if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) {
terrno = TSDB_CODE_RSMA_INVALID_STAT;
return TSDB_CODE_FAILED;
}
@@ -195,11 +207,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
if (param->qmsg[idx]) {
SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
pItem->pRsmaInfo = pRSmaInfo;
- pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle);
+ pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
if (!pItem->taskInfo) {
goto _err;
}
- pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE;
+ pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE;
if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
int64_t msInterval =
convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND);
@@ -211,10 +223,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY;
}
pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2);
- pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
- if (!pItem->tmrHandle) {
- goto _err;
- }
}
return TSDB_CODE_SUCCESS;
_err:
@@ -238,7 +246,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
SMeta *pMeta = pVnode->pMeta;
SMsgCb *pMsgCb = &pVnode->msgCb;
- SRSmaParam *param = &pReq->pRSmaParam;
+ SRSmaParam *param = &pReq->rsmaParam;
if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) {
smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
@@ -251,10 +259,10 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
}
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
- SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
- pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
+ pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
if (pRSmaInfo) {
ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally
smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
@@ -293,16 +301,23 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
goto _err;
}
+
if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) {
goto _err;
}
- if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
+ if (taosHashPut(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
goto _err;
} else {
smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
}
+ // start the persist timer
+ if (TASK_TRIGGER_STAT_INIT ==
+ atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
+ taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
+ }
+
return TSDB_CODE_SUCCESS;
_err:
tdFreeRSmaInfo(pRSmaInfo);
@@ -432,6 +447,16 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
return 0;
}
+static void tdDestroySDataBlockArray(SArray *pArray) {
+#if 0
+ for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+ SSDataBlock *pDataBlock = taosArrayGet(pArray, i);
+ blockDestroyInner(pDataBlock);
+ }
+#endif
+ taosArrayDestroy(pArray);
+}
+
static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
SArray *pResult = NULL;
SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
@@ -462,10 +487,11 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
#if 1
char flag[10] = {0};
snprintf(flag, 10, "level %" PRIi8, pItem->level);
- blockDebugShowData(pResult, flag);
+ blockDebugShowDataBlocks(pResult, flag);
#endif
STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
SSubmitReq *pReq = NULL;
+ // TODO: the schema update should be handled
if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
goto _err;
}
@@ -476,18 +502,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
}
taosMemoryFreeClear(pReq);
+ } else if (terrno == 0) {
+ smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level);
} else {
- smaDebug("vgId:%d, no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno));
+ smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno));
}
- if (blkType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
- atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE);
- }
-
- taosArrayDestroy(pResult);
+ tdDestroySDataBlockArray(pResult);
return TSDB_CODE_SUCCESS;
_err:
- taosArrayDestroy(pResult);
+ tdDestroySDataBlockArray(pResult);
return TSDB_CODE_FAILED;
}
@@ -499,26 +523,38 @@ _err:
*/
static void tdRSmaFetchTrigger(void *param, void *tmrId) {
SRSmaInfoItem *pItem = param;
+ SSma *pSma = pItem->pRsmaInfo->pSma;
+ SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
- if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) {
- smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__,
- taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid);
- SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-
- atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE);
- qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
-
- tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK);
- } else {
- smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__,
- taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid);
+ int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
+ if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
+ smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
+ pItem->level, pItem->pRsmaInfo->suid);
+ return;
}
- // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId);
+ int8_t fetchTriggerStat =
+ atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
+ if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
+ smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
+ pItem->pRsmaInfo->suid);
+
+ tdRefSmaStat(pSma, (SSmaStat *)pStat);
+
+ SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
+ qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
+ tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK);
+
+ tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+
+ } else {
+ smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
+ pItem->pRsmaInfo->suid);
+ }
}
-static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
- tb_uid_t suid, int8_t level) {
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
+ int8_t level) {
if (!pItem || !pItem->taskInfo) {
smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
return TSDB_CODE_SUCCESS;
@@ -533,14 +569,17 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3
}
tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK);
- atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE);
- smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId());
+ atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+ smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma));
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
- taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle);
- taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId);
+ if (pStat->tmrHandle) {
+ taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId);
+ } else {
+ ASSERT(0);
+ }
return TSDB_CODE_SUCCESS;
}
@@ -552,10 +591,10 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
return TSDB_CODE_SUCCESS;
}
- SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
- pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+ pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
@@ -608,7 +647,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
return TSDB_CODE_SUCCESS;
}
-void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) {
+void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) {
tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName);
}
@@ -618,6 +657,11 @@ static void *tdRSmaPersistExec(void *param) {
SSma *pSma = pRSmaStat->pSma;
STfs *pTfs = pSma->pVnode->pTfs;
int64_t toffset = 0;
+ bool isFileCreated = false;
+
+ if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) {
+ goto _end;
+ }
void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
if (!infoHash) {
@@ -625,67 +669,142 @@ static void *tdRSmaPersistExec(void *param) {
}
STFile tFile = {0};
- int32_t vid = 2;
- char qTaskInfoFName[TSDB_FILENAME_LEN];
- tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
- tdInitTFile(&tFile, pTfs, qTaskInfoFName);
- tdCreateTFile(&tFile, pTfs, true, -1);
+ int32_t vid = SMA_VID(pSma);
while (infoHash) {
SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
- char *pOutput = NULL;
- int32_t len = 0;
- if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) {
- smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid,
- terrstr(terrno));
- } else {
- smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid,
- len);
- }
- tdAppendTFile(&tFile, &len, sizeof(len), &toffset);
- tdAppendTFile(&tFile, pOutput, len, &toffset);
- taosMemoryFree(pOutput);
+#if 0
+ smaDebug("table %" PRIi64 " sleep 15s start ...", pRSmaInfo->items[0].pRsmaInfo->suid);
+ for (int32_t i = 15; i > 0; --i) {
+ taosSsleep(1);
+ smaDebug("table %" PRIi64 " countdown %d", pRSmaInfo->items[0].pRsmaInfo->suid, i);
+ }
+ smaDebug("table %" PRIi64 " sleep 15s end ...", pRSmaInfo->items[0].pRsmaInfo->suid);
+#endif
+ for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+ qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo;
+ if (!taskInfo) {
+ smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
+ continue;
+ }
+ char *pOutput = NULL;
+ int32_t len = 0;
+ int8_t type = 0;
+ if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) {
+ smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1,
+ terrstr(terrno));
+ goto _err;
+ } else {
+ if (!pOutput) {
+ smaDebug("vgId:%d, table %" PRIi64
+ " level %d serialize rsma task success but no output(len %d) and no need to persist",
+ vid, pRSmaInfo->suid, i + 1, len);
+ continue;
+ } else if (len <= 0) {
+ smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and no need to persist",
+ vid, pRSmaInfo->suid, i + 1, len);
+ taosMemoryFree(pOutput);
+ }
+ smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and need persist", vid,
+ pRSmaInfo->suid, i + 1, len);
+#if 1
+ if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) {
+ smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid,
+ i + 1, terrstr(terrno));
+ } else {
+ smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1);
+ }
+#endif
+ }
+
+ if (!isFileCreated) {
+ char qTaskInfoFName[TSDB_FILENAME_LEN];
+ tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
+ tdInitTFile(&tFile, pTfs, qTaskInfoFName);
+ tdCreateTFile(&tFile, pTfs, true, -1);
+
+ isFileCreated = true;
+ }
+ len += (sizeof(len) + sizeof(pRSmaInfo->suid));
+ tdAppendTFile(&tFile, &len, sizeof(len), &toffset);
+ tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset);
+ tdAppendTFile(&tFile, pOutput, len, &toffset);
+
+ taosMemoryFree(pOutput);
+ }
infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
}
-_end:
+_normal:
+ if (isFileCreated) {
+ if (tdUpdateTFileHeader(&tFile) < 0) {
+ smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno));
+ tdCloseTFile(&tFile);
+ tdRemoveTFile(&tFile);
+ goto _err;
+ } else {
+ smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_FILE_FULL_NAME(&tFile));
+ }
- if (tdUpdateTFileHeader(&tFile) < 0) {
- smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno));
tdCloseTFile(&tFile);
- tdRemoveTFile(&tFile);
- return NULL;
+
+ char newFName[TSDB_FILENAME_LEN];
+ strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
+ char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]);
+ strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
+ if (taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName) != 0) {
+ smaError("vgId:%d, failed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName);
+ goto _err;
+ } else {
+ smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName);
+ }
}
-
- tdCloseTFile(&tFile);
-
- char newFName[TSDB_FILENAME_LEN];
- strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
- char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]);
- strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
- taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName);
-
- atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE);
- return NULL;
+ goto _end;
_err:
- atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE);
- // remove the .tmp file
+ if (isFileCreated) {
+ tdRemoveTFile(&tFile);
+ }
+_end:
+ if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
+ TASK_TRIGGER_STAT_INACTIVE,
+ TASK_TRIGGER_STAT_ACTIVE)) {
+ smaDebug("vgId:%d, persist task is active again", vid);
+ } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
+ TASK_TRIGGER_STAT_CANCELLED,
+ TASK_TRIGGER_STAT_FINISHED)) {
+ smaDebug("vgId:%d, persist task is cancelled", vid);
+ } else {
+ smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
+ ASSERT(0);
+ }
+ atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
+ taosThreadExit(NULL);
return NULL;
}
static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
- smaWarn("%s:%d entry ", __func__, __LINE__);
- TdThread threadId;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED);
+ TdThread tid;
- if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) {
- smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno));
+ if (taosThreadCreate(&tid, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) {
+ if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
+ TASK_TRIGGER_STAT_INACTIVE,
+ TASK_TRIGGER_STAT_ACTIVE)) {
+ smaDebug("persist task is active again");
+ } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
+ TASK_TRIGGER_STAT_CANCELLED,
+ TASK_TRIGGER_STAT_FINISHED)) {
+ smaDebug(" persist task is cancelled and set finished");
+ } else {
+ smaWarn("persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
+ ASSERT(0);
+ }
+ atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
}
taosThreadAttrDestroy(&thAttr);
- smaWarn("%s:%d end ", __func__, __LINE__);
}
/**
@@ -696,17 +815,34 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
*/
static void tdRSmaPersistTrigger(void *param, void *tmrId) {
SRSmaStat *pRSmaStat = param;
-
- if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) {
- smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId());
- atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE);
-
- // execution
- tdRSmaPersistTask(pRSmaStat);
- } else {
- smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__,
- taosGetSelfPthreadId());
+ int8_t tmrStat =
+ atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
+ switch (tmrStat) {
+ case TASK_TRIGGER_STAT_ACTIVE: {
+ atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 1);
+ if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
+ TASK_TRIGGER_STAT_CANCELLED,
+ TASK_TRIGGER_STAT_FINISHED)) {
+ smaDebug("rsma persistence start since active");
+ tdRSmaPersistTask(pRSmaStat);
+ taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId);
+ } else {
+ atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
+ }
+ } break;
+ case TASK_TRIGGER_STAT_CANCELLED: {
+ atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
+ smaDebug("rsma persistence not start since cancelled and finished");
+ } break;
+ case TASK_TRIGGER_STAT_INACTIVE: {
+ smaDebug("rsma persistence not start since inactive");
+ } break;
+ case TASK_TRIGGER_STAT_INIT: {
+ smaDebug("rsma persistence not start since init");
+ } break;
+ default: {
+ smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
+ ASSERT(0);
+ } break;
}
-
- taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId);
}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c
index 76690a3bde..85fddebf55 100644
--- a/source/dnode/vnode/src/sma/smaUtil.c
+++ b/source/dnode/vnode/src/sma/smaUtil.c
@@ -15,6 +15,8 @@
#include "sma.h"
+// smaFileUtil ================
+
#define TD_FILE_HEAD_SIZE 512
#define TD_FILE_STATE_OK 0
@@ -32,7 +34,7 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) {
tlen += taosEncodeFixedU32(buf, pInfo->magic);
tlen += taosEncodeFixedU32(buf, pInfo->ftype);
tlen += taosEncodeFixedU32(buf, pInfo->fver);
- tlen += taosEncodeFixedU64(buf, pInfo->fsize);
+ tlen += taosEncodeFixedI64(buf, pInfo->fsize);
return tlen;
}
@@ -41,7 +43,7 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) {
buf = taosDecodeFixedU32(buf, &(pInfo->magic));
buf = taosDecodeFixedU32(buf, &(pInfo->ftype));
buf = taosDecodeFixedU32(buf, &(pInfo->fver));
- buf = taosDecodeFixedU64(buf, &(pInfo->fsize));
+ buf = taosDecodeFixedI64(buf, &(pInfo->fsize));
return buf;
}
@@ -235,4 +237,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp
return 0;
}
-int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); }
\ No newline at end of file
+int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); }
+
+// smaXXXUtil ================
+// ...
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index be8fef1249..afeeeab500 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -112,7 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
- if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
+ if (tqRetrieveDataBlock(&block, pReader) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0);
}
@@ -129,7 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
SSDataBlock block = {0};
- if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
+ if (tqRetrieveDataBlock(&block, pReader) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0);
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 6184d8f810..cbee639911 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -109,11 +109,15 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
}
bool tqNextDataBlock(STqReadHandle* pHandle) {
+ if (pHandle->pMsg == NULL) return false;
while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
return false;
}
- if (pHandle->pBlock == NULL) return false;
+ if (pHandle->pBlock == NULL) {
+ pHandle->pMsg = NULL;
+ return false;
+ }
if (pHandle->tbIdHash == NULL) {
return true;
@@ -142,10 +146,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
return false;
}
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
- int32_t* pNumOfRows) {
- *pUid = 0;
-
+int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
// TODO: cache multiple schema
int32_t sversion = htonl(pHandle->pBlock->sversion);
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
@@ -176,7 +177,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
STSchema* pTschema = pHandle->pSchema;
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
- *pNumOfRows = pHandle->msgIter.numOfRows;
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
if (colNumNeed == 0) {
@@ -217,22 +217,22 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
}
}
- if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) {
+ if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) {
goto FAIL;
}
int32_t colActual = blockDataGetNumOfCols(pBlock);
- // TODO in stream shuffle case, fetch groupId
- *pGroupId = 0;
-
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
int32_t curRow = 0;
tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
- *pUid = pHandle->msgIter.uid; // set the uid of table for submit block
+
+ pBlock->info.groupId = 0;
+ pBlock->info.uid = pHandle->msgIter.uid; // set the uid of table for submit block
+ pBlock->info.rows = pHandle->msgIter.numOfRows;
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
tdSTSRowIterReset(&iter, row);
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index ef3b205b3e..0bb9918488 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -18,57 +18,87 @@
SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid,
const char* stbFullName, int32_t vgId) {
SSubmitReq* ret = NULL;
+ SArray* schemaReqs = NULL;
+ SArray* schemaReqSz = NULL;
SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
if (!tagArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
- // cal size
- int32_t cap = sizeof(SSubmitReq);
int32_t sz = taosArrayGetSize(pBlocks);
- for (int32_t i = 0; i < sz; i++) {
- SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
- int32_t rows = pDataBlock->info.rows;
- // TODO min
- int32_t rowSize = pDataBlock->info.rowSize;
- int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
- int32_t schemaLen = 0;
- if (createTb) {
- SVCreateTbReq createTbReq = {0};
- char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
- createTbReq.name = cname;
- createTbReq.flags = 0;
- createTbReq.type = TSDB_CHILD_TABLE;
- createTbReq.ctb.suid = suid;
-
- STagVal tagVal = {
- .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
- .type = TSDB_DATA_TYPE_UBIGINT,
- .i64 = (int64_t)pDataBlock->info.groupId,
+ if (createTb) {
+ schemaReqs = taosArrayInit(sz, sizeof(void*));
+ schemaReqSz = taosArrayInit(sz, sizeof(int32_t));
+ for (int32_t i = 0; i < sz; i++) {
+ SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+ STagVal tagVal = {
+ .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
+ .type = TSDB_DATA_TYPE_UBIGINT,
+ .i64 = (int64_t)pDataBlock->info.groupId,
};
STag* pTag = NULL;
taosArrayClear(tagArray);
taosArrayPush(tagArray, &tagVal);
tTagNew(tagArray, 1, false, &pTag);
if (pTag == NULL) {
- tdDestroySVCreateTbReq(&createTbReq);
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(tagArray);
return NULL;
}
+
+ SVCreateTbReq createTbReq = {0};
+ createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
+ createTbReq.flags = 0;
+ createTbReq.type = TSDB_CHILD_TABLE;
+ createTbReq.ctb.suid = suid;
createTbReq.ctb.pTag = (uint8_t*)pTag;
int32_t code;
+ int32_t schemaLen;
tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
-
- tdDestroySVCreateTbReq(&createTbReq);
if (code < 0) {
+ tdDestroySVCreateTbReq(&createTbReq);
taosArrayDestroy(tagArray);
+ taosMemoryFreeClear(ret);
return NULL;
}
- }
+ void* schemaStr = taosMemoryMalloc(schemaLen);
+ if (schemaStr == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+ taosArrayPush(schemaReqs, &schemaStr);
+ taosArrayPush(schemaReqSz, &schemaLen);
+
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, schemaStr, schemaLen);
+ code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
+ if (code < 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+ tEncoderClear(&encoder);
+ tdDestroySVCreateTbReq(&createTbReq);
+ }
+ }
+ taosArrayDestroy(tagArray);
+
+ // cal size
+ int32_t cap = sizeof(SSubmitReq);
+ for (int32_t i = 0; i < sz; i++) {
+ SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+ int32_t rows = pDataBlock->info.rows;
+ // TODO min
+ int32_t rowSize = pDataBlock->info.rowSize;
+ int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+
+ int32_t schemaLen = 0;
+ if (createTb) {
+ schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
+ }
cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen;
}
@@ -99,55 +129,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
int32_t schemaLen = 0;
if (createTb) {
- SVCreateTbReq createTbReq = {0};
- char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
- createTbReq.name = cname;
- createTbReq.flags = 0;
- createTbReq.type = TSDB_CHILD_TABLE;
- createTbReq.ctb.suid = suid;
-
- STagVal tagVal = {
- .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
- .type = TSDB_DATA_TYPE_UBIGINT,
- .i64 = (int64_t)pDataBlock->info.groupId,
- };
- taosArrayClear(tagArray);
- taosArrayPush(tagArray, &tagVal);
- STag* pTag = NULL;
- tTagNew(tagArray, 1, false, &pTag);
- if (pTag == NULL) {
- tdDestroySVCreateTbReq(&createTbReq);
- taosArrayDestroy(tagArray);
- taosMemoryFreeClear(ret);
- return NULL;
- }
- createTbReq.ctb.pTag = (uint8_t*)pTag;
-
- int32_t code;
- tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
- if (code < 0) {
- tdDestroySVCreateTbReq(&createTbReq);
- taosArrayDestroy(tagArray);
- taosMemoryFreeClear(ret);
- return NULL;
- }
-
- SEncoder encoder = {0};
- tEncoderInit(&encoder, blkSchema, schemaLen);
- code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
- tEncoderClear(&encoder);
- tdDestroySVCreateTbReq(&createTbReq);
-
- if (code < 0) {
- taosArrayDestroy(tagArray);
- taosMemoryFreeClear(ret);
- return NULL;
- }
+ schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
+ void* schemaStr = taosArrayGetP(schemaReqs, i);
+ memcpy(blkSchema, schemaStr, schemaLen);
}
blkHead->schemaLen = htonl(schemaLen);
STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen);
-
for (int32_t j = 0; j < rows; j++) {
SRowBuilder rb = {0};
tdSRowInit(&rb, pTSchema->version);
@@ -175,7 +163,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
}
ret->length = htonl(ret->length);
- taosArrayDestroy(tagArray);
+
+ if (schemaReqs) taosArrayDestroyP(schemaReqs, taosMemoryFree);
+ taosArrayDestroy(schemaReqSz);
+
return ret;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 540810f876..03f18cc766 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2852,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
break;
}
- STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
+ STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0};
taosArrayPush(list, &info);
}
@@ -2876,6 +2876,33 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
return TSDB_CODE_SUCCESS;
}
+/**
+ * @brief Get all suids since suid
+ *
+ * @param pMeta
+ * @param suid return all suids in one vnode if suid is 0
+ * @param list
+ * @return int32_t
+ */
+int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
+ SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid);
+ if(!pCur) {
+ return TSDB_CODE_FAILED;
+ }
+
+ while (1) {
+ tb_uid_t id = metaStbCursorNext(pCur);
+ if (id == 0) {
+ break;
+ }
+
+ taosArrayPush(list, &id);
+ }
+
+ metaCloseStbCursor(pCur);
+ return TSDB_CODE_SUCCESS;
+}
+
static void destroyHelper(void* param) {
if (param == NULL) {
return;
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 3715866bb8..dbbe3d3750 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -72,7 +72,12 @@ int vnodeBegin(SVnode *pVnode) {
return 0;
}
-int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; }
+int vnodeShouldCommit(SVnode *pVnode) {
+ if (pVnode->inUse) {
+ return pVnode->inUse->size > pVnode->config.szBuf / 3;
+ }
+ return false;
+}
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
char fname[TSDB_FILENAME_LEN];
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index dfc258b42b..4d73dbc406 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -152,12 +152,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
return pVnode;
_err:
+ if (pVnode->pSma) smaClose(pVnode->pSma);
if (pVnode->pQuery) vnodeQueryClose(pVnode);
if (pVnode->pTq) tqClose(pVnode->pTq);
if (pVnode->pWal) walClose(pVnode->pWal);
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
if (pVnode->pMeta) metaClose(pVnode->pMeta);
- if (pVnode->pSma) smaClose(pVnode->pSma);
+
tsem_destroy(&(pVnode->canCommit));
taosMemoryFree(pVnode);
@@ -166,13 +167,14 @@ _err:
void vnodeClose(SVnode *pVnode) {
if (pVnode) {
+ smaCloseEnv(pVnode->pSma);
vnodeCommit(pVnode);
vnodeSyncClose(pVnode);
vnodeQueryClose(pVnode);
walClose(pVnode->pWal);
tqClose(pVnode->pTq);
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
- smaClose(pVnode->pSma);
+ smaCloseEx(pVnode->pSma);
metaClose(pVnode->pMeta);
vnodeCloseBufPool(pVnode);
// destroy handle
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 88bdea3ae7..72c766e0ae 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -299,7 +299,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp
void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
// TODO
- blockDebugShowData(data, __func__);
+ blockDebugShowDataBlocks(data, __func__);
tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 4c56180979..b998f40195 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -137,6 +137,25 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr());
} else {
code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType));
+ if (code == 1) {
+ do {
+ static int32_t cnt = 0;
+ if (cnt++ % 1000 == 1) {
+ vInfo("vgId:%d, msg:%p apply right now, apply index:%ld, msgtype:%s,%d", vgId, pMsg,
+ pMsg->info.conn.applyIndex, TMSG_INFO(pMsg->msgType), pMsg->msgType);
+ }
+ } while (0);
+
+ SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
+ if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) {
+ rsp.code = terrno;
+ vInfo("vgId:%d, msg:%p failed to apply right now since %s", vgId, pMsg, terrstr());
+ }
+
+ if (rsp.info.handle != NULL) {
+ tmsgSendRsp(&rsp);
+ }
+ }
}
}
@@ -163,10 +182,12 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
tmsgSendRedirectRsp(&rsp, &newEpSet);
} else {
- if (terrno != 0) code = terrno;
- vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
- SRpcMsg rsp = {.code = code, .info = pMsg->info};
- tmsgSendRsp(&rsp);
+ if (code != 1) {
+ if (terrno != 0) code = terrno;
+ vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
+ SRpcMsg rsp = {.code = code, .info = pMsg->info};
+ tmsgSendRsp(&rsp);
+ }
}
vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
@@ -260,7 +281,7 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
- ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+ ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL);
syncClientRequestDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
@@ -359,34 +380,18 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c
SyncIndex beginIndex = SYNC_INDEX_INVALID;
char logBuf[256] = {0};
- if (pFsm->FpGetSnapshotInfo != NULL) {
- (*pFsm->FpGetSnapshotInfo)(pFsm, &snapshot);
- beginIndex = snapshot.lastApplyIndex;
- }
+ snprintf(logBuf, sizeof(logBuf),
+ "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
+ pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
+ beginIndex);
+ syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
- if (cbMeta.index > beginIndex) {
- snprintf(
- logBuf, sizeof(logBuf),
- "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
- pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
- syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
-
- SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
- rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
- memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
- syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
- rpcMsg.info.conn.applyIndex = cbMeta.index;
- tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
-
- } else {
- char logBuf[256] = {0};
- snprintf(logBuf, sizeof(logBuf),
- "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
- "beginIndex :%ld\n",
- pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
- beginIndex);
- syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
- }
+ SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
+ rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+ memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
+ syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
+ rpcMsg.info.conn.applyIndex = cbMeta.index;
+ tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
}
static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index d8d231e952..1cc3f9b874 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -293,6 +293,7 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_RES,
STREAM_SCAN_FROM_UPDATERES,
STREAM_SCAN_FROM_DATAREADER,
+ STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
} EStreamScanMode;
typedef struct SCatchSupporter {
@@ -348,7 +349,9 @@ typedef struct SStreamBlockScanInfo {
SArray* childIds;
SessionWindowSupporter sessionSup;
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
- int32_t scanWinIndex;
+ int32_t scanWinIndex; // for state operator
+ int32_t pullDataResIndex;
+ SSDataBlock* pPullDataRes; // pull data SSDataBlock
} SStreamBlockScanInfo;
typedef struct SSysTableScanInfo {
@@ -400,7 +403,7 @@ typedef struct SIntervalAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; // basic info
SAggSupporter aggSup; // aggregate supporter
-
+ SExprSupp scalarSupp; // supporter for perform scalar function
SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
@@ -427,8 +430,13 @@ typedef struct SStreamFinalIntervalOperatorInfo {
STimeWindowAggSupp twAggSup;
SArray* pChildren;
SSDataBlock* pUpdateRes;
+ bool returnUpdate;
SPhysiNode* pPhyNode; // create new child
bool isFinal;
+ SHashObj* pPullDataMap;
+ SArray* pPullWins; // SPullWindowInfo
+ int32_t pullIndex;
+ SSDataBlock* pPullDataRes;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo {
@@ -722,15 +730,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
-
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock,
- SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
- SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
- STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream);
+ STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream);
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
@@ -800,9 +805,10 @@ int32_t getMaximumIdleDurationSec();
* ops: root operator
* data: *data save the result of encode, need to be freed by caller
* length: *length save the length of *data
+ * nOptrWithVal: *nOptrWithVal save the number of optr with value
* return: result code, 0 means success
*/
-int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length);
+int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal);
/*
* ops: root operator, created by caller
@@ -851,6 +857,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
+SSDataBlock* createPullDataBlock();
#ifdef __cplusplus
}
diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h
index 35bdd4ee55..972f4469e4 100644
--- a/source/libs/executor/inc/tsort.h
+++ b/source/libs/executor/inc/tsort.h
@@ -136,6 +136,13 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
*/
void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
+/**
+ *
+ * @param pVHandle
+ * @return
+ */
+uint64_t tsortGetGroupId(STupleHandle* pVHandle);
+
/**
*
* @param pSortHandle
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index f23a6fded6..7a601bae83 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -159,7 +159,12 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
}
SArray* createSortInfo(SNodeList* pNodeList) {
- size_t numOfCols = LIST_LENGTH(pNodeList);
+ size_t numOfCols = 0;
+ if (pNodeList != NULL) {
+ numOfCols = LIST_LENGTH(pNodeList);
+ } else {
+ numOfCols = 0;
+ }
SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo));
if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -316,7 +321,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
for (int i = 0; i < taosArrayGetSize(res); i++) {
- STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)};
+ STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
taosArrayDestroy(res);
@@ -344,7 +349,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
}
}else { // Create one table group.
- STableKeyInfo info = {.lastKey = 0, .uid = tableUid};
+ STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c
index a9e1e03178..e4c0959185 100644
--- a/source/libs/executor/src/executorMain.c
+++ b/source/libs/executor/src/executorMain.c
@@ -222,7 +222,13 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
return TSDB_CODE_INVALID_PARA;
}
- return encodeOperator(pTaskInfo->pRoot, pOutput, len);
+ int32_t nOptrWithVal = 0;
+ int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
+ if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
+ taosMemoryFreeClear(*pOutput);
+ *len = 0;
+ }
+ return code;
}
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 7d14e6a7d2..2863f1139c 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -2928,6 +2928,13 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len
int32_t totalSize =
sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize);
+ // no result
+ if (getTotalBufSize(pSup->pResultBuf) == 0) {
+ *result = NULL;
+ *length = 0;
+ return TSDB_CODE_SUCCESS;
+ }
+
*result = (char*)taosMemoryCalloc(1, totalSize);
if (*result == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -4034,6 +4041,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
int32_t len = (int32_t)(pStart - (char*)keyBuf);
uint64_t groupId = calcGroupId(keyBuf, len);
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t));
+ info->groupId = groupId;
groupNum++;
nodesClearList(groupNew);
@@ -4137,7 +4145,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
} else { // Create one table group.
- STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid};
+ STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0};
taosArrayPush(pTableListInfo->pTableList, &info);
}
@@ -4227,8 +4235,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type);
- pOptr =
- createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream);
+ pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
+ pTaskInfo, isStream);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
@@ -4272,17 +4280,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) {
SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode;
-
- SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc;
- SSDataBlock* pResBlock = createResDataBlock(pDescNode);
-
- SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
- int32_t numOfOutputCols = 0;
- SArray* pColList =
- extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
- SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
- SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
- pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo);
+ pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) {
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
@@ -4475,12 +4473,12 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
return 0;
}
-int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
+int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
int32_t code = TDB_CODE_SUCCESS;
char* pCurrent = NULL;
int32_t currLength = 0;
if (ops->fpSet.encodeResultRow) {
- if (result == NULL || length == NULL) {
+ if (result == NULL || length == NULL || nOptrWithVal == NULL) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength);
@@ -4491,8 +4489,15 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
*result = NULL;
}
return code;
+ } else if (currLength == 0) {
+ ASSERT(!pCurrent);
+ goto _downstream;
}
+ ++(*nOptrWithVal);
+
+ ASSERT(currLength >= 0);
+
if (*result == NULL) {
*result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t));
if (*result == NULL) {
@@ -4518,8 +4523,9 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
*length = *(int32_t*)(*result);
}
+_downstream:
for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
- code = encodeOperator(ops->pDownstream[i], result, length);
+ code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
if (code != TDB_CODE_SUCCESS) {
return code;
}
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7affac76d2..c51ef44154 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -448,7 +448,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
- qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
+ qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
// do prepare for the next round table scan operation
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
@@ -467,7 +467,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
- qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
+ qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
while (pTableScanInfo->scanTimes < total) {
@@ -492,7 +492,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
GET_TASKID(pTaskInfo));
for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
- qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
+ qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0;
@@ -507,20 +507,21 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- if(pInfo->currentGroupId == -1){
+ if (pInfo->currentGroupId == -1) {
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
- SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
+ SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbCleanupReadHandle(pInfo->dataReader);
- tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
+ tsdbReaderT* pReader =
+ tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
pInfo->dataReader = pReader;
}
SSDataBlock* result = doTableScanGroup(pOperator);
- if(result){
+ if (result) {
return result;
}
@@ -530,7 +531,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
return NULL;
}
- SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
+ SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbSetTableList(pInfo->dataReader, tableList);
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
@@ -538,7 +539,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
pInfo->scanTimes = 0;
result = doTableScanGroup(pOperator);
- if(result){
+ if (result) {
return result;
}
@@ -776,15 +777,14 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
}
-static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
- SSDataBlock* pSDB = pInfo->pUpdateRes;
- STimeWindow win = {
- .skey = INT64_MIN,
- .ekey = INT64_MAX,
+static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
+ STimeWindow win = {
+ .skey = INT64_MIN,
+ .ekey = INT64_MAX,
};
bool needRead = false;
- if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
- SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex);
+ if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) {
+ SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex);
TSKEY* tsCols = (TSKEY*)pColDataInfo->pData;
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
@@ -793,15 +793,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
int64_t gap = pInfo->sessionSup.gap;
int32_t winIndex = 0;
SResultWindowInfo* pCurWin =
- getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
+ getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
win = pCurWin->win;
- pInfo->updateResIndex +=
- updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL);
+ (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL);
} else {
- win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval,
- pInfo->interval.precision, NULL);
- pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey,
- binarySearchForKey, NULL, TSDB_ORDER_ASC);
+ win =
+ getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+ (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
+ TSDB_ORDER_ASC);
}
needRead = true;
} else if (isStateWindow(pInfo)) {
@@ -822,7 +821,10 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
pTableScanInfo->cond.twindows[0] = win;
pTableScanInfo->curTWinIdx = 0;
-// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+ // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+ // if (!pTableScanInfo->dataReader) {
+ // return false;
+ // }
pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1;
return true;
@@ -862,12 +864,12 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_
*/
}
-static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) {
+static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
while (1) {
SSDataBlock* pResult = NULL;
pResult = doTableScan(pInfo->pSnapshotReadOp);
if (pResult == NULL) {
- if (prepareDataScan(pInfo)) {
+ if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) {
// scan next window data
pResult = doTableScan(pInfo->pSnapshotReadOp);
}
@@ -916,7 +918,7 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
pUpdateBlock->info.rows = i;
pInfo->tsArrayIndex += i;
pUpdateBlock->info.groupId = pInfo->groupId;
- pUpdateBlock->info.type = STREAM_REPROCESS;
+ pUpdateBlock->info.type = STREAM_CLEAR;
blockDataUpdateTsWindow(pUpdateBlock, 0);
}
// all rows have same group id
@@ -970,6 +972,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
int32_t current = pInfo->validBlockIndex++;
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
blockDataUpdateTsWindow(pBlock, 0);
+ if (pBlock->info.type == STREAM_RETRIEVE) {
+ pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK;
+ pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
+ copyDataBlock(pInfo->pPullDataRes, pBlock);
+ pInfo->pullDataResIndex = 0;
+ prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+ updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
+ }
return pBlock;
} else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
@@ -979,28 +989,39 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
if (!isStateWindow(pInfo)) {
- prepareDataScan(pInfo);
+ prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
}
return pInfo->pUpdateRes;
+ } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) {
+ SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+ if (pSDB != NULL) {
+ getUpdateDataBlock(pInfo, true, pSDB, NULL);
+ pSDB->info.type = STREAM_PUSH_DATA;
+ return pSDB;
+ }
+ pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
} else {
if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
- prepareDataScan(pInfo);
+ prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
}
if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
- SSDataBlock* pSDB = doDataScan(pInfo);
+ SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
if (pSDB == NULL) {
setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
if (pInfo->pUpdateRes->info.rows > 0) {
if (!isStateWindow(pInfo)) {
- prepareDataScan(pInfo);
+ // Todo(liuyao) mybe can delete this.
+ bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
+ ASSERT(test == false);
}
return pInfo->pUpdateRes;
} else {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
}
} else {
+ pSDB->info.type = STREAM_NORMAL;
getUpdateDataBlock(pInfo, true, pSDB, NULL);
return pSDB;
}
@@ -1012,12 +1033,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
while (tqNextDataBlock(pInfo->streamBlockReader)) {
SSDataBlock block = {0};
- uint64_t groupId = 0;
- uint64_t uid = 0;
- int32_t numOfRows = 0;
// todo refactor
- int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows);
+ int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader);
+
+ uint64_t groupId = block.info.groupId;
+ uint64_t uid = block.info.uid;
+ int32_t numOfRows = block.info.rows;
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
pTaskInfo->code = code;
@@ -1070,10 +1092,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
taosArrayDestroy(block.pDataBlock);
if (pInfo->pRes->pDataBlock == NULL) {
// TODO add log
+ updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
pOperator->status = OP_EXEC_DONE;
pTaskInfo->code = terrno;
return NULL;
}
+
// currently only the tbname pseudo column
if (pInfo->numOfPseudoExpr > 0) {
addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
@@ -1091,12 +1115,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pOperator->resultInfo.totalRows += pBlockInfo->rows;
if (pBlockInfo->rows == 0) {
+ updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
pOperator->status = OP_EXEC_DONE;
} else if (pInfo->pUpdateInfo) {
pInfo->tsArrayIndex = 0;
getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes);
if (pInfo->pUpdateRes->info.rows > 0) {
- if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) {
+ if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) {
pInfo->updateResIndex = 0;
pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
} else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) {
@@ -1130,9 +1155,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
return tableIdList;
}
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
- STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo,
- STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) {
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
+ SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
+ uint64_t taskId) {
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@@ -1209,6 +1234,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
pInfo->groupId = 0;
+ pInfo->pPullDataRes = createPullDataBlock();
pOperator->name = "StreamBlockScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
@@ -1718,14 +1744,14 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
int32_t num = 0;
- SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
+ SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
- pInfo->accountId = pScanPhyNode->accountId;
- pInfo->pUser = taosMemoryStrDup((void*) pUser);
+ pInfo->accountId = pScanPhyNode->accountId;
+ pInfo->pUser = taosMemoryStrDup((void*)pUser);
pInfo->showRewrite = pScanPhyNode->showRewrite;
- pInfo->pRes = pResBlock;
- pInfo->pCondition = pScanNode->node.pConditions;
- pInfo->scanCols = colList;
+ pInfo->pRes = pResBlock;
+ pInfo->pCondition = pScanNode->node.pConditions;
+ pInfo->scanCols = colList;
initResultSizeInfo(pOperator, 4096);
@@ -1741,13 +1767,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pInfo->readHandle = *(SReadHandle*)readHandle;
}
- pOperator->name = "SysTableScanOperator";
+ pOperator->name = "SysTableScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN;
- pOperator->blocking = false;
- pOperator->status = OP_NOT_OPENED;
- pOperator->info = pInfo;
+ pOperator->blocking = false;
+ pOperator->status = OP_NOT_OPENED;
+ pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock);
- pOperator->pTaskInfo = pTaskInfo;
+ pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL);
@@ -1934,11 +1960,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
- pInfo->pTableList = pTableListInfo;
- pInfo->pColMatchInfo = colList;
- pInfo->pRes = createResDataBlock(pDescNode);
- pInfo->readHandle = *pReadHandle;
- pInfo->curPos = 0;
+ pInfo->pTableList = pTableListInfo;
+ pInfo->pColMatchInfo = colList;
+ pInfo->pRes = createResDataBlock(pDescNode);
+ pInfo->readHandle = *pReadHandle;
+ pInfo->curPos = 0;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
@@ -1965,7 +1991,10 @@ _error:
typedef struct STableMergeScanInfo {
STableListInfo* tableListInfo;
- int32_t currentGroupId;
+ int32_t tableStartIndex;
+ int32_t tableEndIndex;
+ bool hasGroupId;
+ uint64_t groupId;
SArray* dataReaders; // array of tsdbReaderT*
SReadHandle readHandle;
@@ -2022,7 +2051,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
return TSDB_CODE_SUCCESS;
}
- code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags);
+ code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -2030,6 +2059,22 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
return TSDB_CODE_SUCCESS;
}
+int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
+ int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
+ uint64_t taskId) {
+ for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
+ SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
+ taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
+
+ tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId);
+ taosArrayPush(arrayReader, &pReader);
+
+ taosArrayDestroy(subTableList);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
// todo refactor
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@@ -2216,34 +2261,32 @@ SArray* generateSortByTsInfo(int32_t order) {
return pList;
}
-static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId,
- uint64_t taskId) {
- for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) {
- SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo));
- taosArrayPush(tmp, taosArrayGet(tableList, i));
-
- tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId);
- taosArrayPush(arrayReader, &pReader);
-
- taosArrayDestroy(tmp);
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId);
+ {
+ size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
+ int32_t i = pInfo->tableStartIndex + 1;
+ for (; i < tableListSize; ++i) {
+ STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
+ if (tableKeyInfo->groupId != pInfo->groupId) {
+ break;
+ }
+ }
+ pInfo->tableEndIndex = i - 1;
+ }
- createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList,
+ int32_t tableStartIdx = pInfo->tableStartIndex;
+ int32_t tableEndIdx = pInfo->tableEndIndex;
+
+ STableListInfo* tableListInfo = pInfo->tableListInfo;
+ createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
pInfo->dataReaders, pInfo->queryId, pInfo->taskId);
// todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result
- int32_t tableLen = taosArrayGetSize(tableList);
- pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1);
+ pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str);
@@ -2330,43 +2373,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
+ size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
+ if (!pInfo->hasGroupId) {
+ pInfo->hasGroupId = true;
- if (pInfo->currentGroupId == -1) {
- pInfo->currentGroupId++;
- if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) {
+ if (tableListSize == 0) {
doSetOperatorCompleted(pOperator);
return NULL;
}
+ pInfo->tableStartIndex = 0;
+ pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
- SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
- if (pBlock != NULL) {
- uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t));
- if(groupId) pBlock->info.groupId = *groupId;
-
- pOperator->resultInfo.totalRows += pBlock->info.rows;
- return pBlock;
+ SSDataBlock* pBlock = NULL;
+ while (pInfo->tableStartIndex < tableListSize) {
+ pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
+ if (pBlock != NULL) {
+ pBlock->info.groupId = pInfo->groupId;
+ pOperator->resultInfo.totalRows += pBlock->info.rows;
+ return pBlock;
+ } else {
+ stopGroupTableMergeScan(pOperator);
+ if (pInfo->tableEndIndex >= tableListSize - 1) {
+ doSetOperatorCompleted(pOperator);
+ break;
+ }
+ pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
+ pInfo->groupId =
+ ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
+ startGroupTableMergeScan(pOperator);
+ }
}
- stopGroupTableMergeScan(pOperator);
- pInfo->currentGroupId++;
- if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) {
- doSetOperatorCompleted(pOperator);
- return NULL;
- }
- startGroupTableMergeScan(pOperator);
-
- pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
- if (pBlock != NULL) {
- uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t));
- if(groupId) pBlock->info.groupId = *groupId;
-
- pOperator->resultInfo.totalRows += pBlock->info.rows;
- return pBlock;
- }
-
- doSetOperatorCompleted(pOperator);
-
return pBlock;
}
@@ -2403,6 +2441,12 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
return TSDB_CODE_SUCCESS;
}
+int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
+ const STableKeyInfo* info1 = p1;
+ const STableKeyInfo* info2 = p2;
+ return info1->groupId - info2->groupId;
+}
+
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
uint64_t taskId) {
@@ -2411,6 +2455,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
+ if (pTableScanNode->pGroupTags) {
+ taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
+ }
SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
@@ -2443,7 +2490,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
pInfo->queryId = queryId;
pInfo->taskId = taskId;
- pInfo->currentGroupId = -1;
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 09cc4e47be..44ff4c1c90 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -142,7 +142,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
SSDataBlock* loadNextDataBlock(void* param) {
SOperatorInfo* pOperator = (SOperatorInfo*)param;
- return pOperator->fpSet.getNextFn(pOperator);
+ SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator);
+ return pBlock;
}
// todo refactor: merged with fetch fp
@@ -442,7 +443,7 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo) {
SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo));
- SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) {
goto _error;
}
@@ -474,8 +475,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->pTaskInfo = pTaskInfo;
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL,
- NULL, getGroupSortExplainExecInfo);
+ pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo,
+ NULL, NULL, getGroupSortExplainExecInfo);
int32_t code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -493,7 +494,7 @@ _error:
//=====================================================================================
// Multiway Sort Merge operator
-typedef struct SMultiwaySortMergeOperatorInfo {
+typedef struct SMultiwayMergeOperatorInfo {
SOptrBasicInfo binfo;
int32_t bufPageSize;
@@ -503,14 +504,17 @@ typedef struct SMultiwaySortMergeOperatorInfo {
SSortHandle* pSortHandle;
SArray* pColMatchInfo; // for index map from table scan output
- SSDataBlock* pInputBlock;
- int64_t startTs; // sort start time
- uint64_t groupId;
-} SMultiwaySortMergeOperatorInfo;
+ SSDataBlock* pInputBlock;
+ int64_t startTs; // sort start time
+ bool groupSort;
+ bool hasGroupId;
+ uint64_t groupId;
+ STupleHandle* prefetchedTuple;
+} SMultiwayMergeOperatorInfo;
-int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
- SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
+ SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (OPTR_IS_OPENED(pOperator)) {
return TSDB_CODE_SUCCESS;
@@ -524,8 +528,9 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
pInfo->pInputBlock, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL);
- tsortSetCompareGroupId(pInfo->pSortHandle, true);
-
+
+ tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort);
+
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
ps->param = pOperator->pDownstream[i];
@@ -547,8 +552,8 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
SArray* pColMatchInfo, SOperatorInfo* pOperator) {
- SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
blockDataCleanup(pDataBlock);
@@ -560,18 +565,47 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataEnsureCapacity(p, capacity);
while (1) {
- STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
+ STupleHandle* pTupleHandle = NULL;
+ if (pInfo->groupSort) {
+ if (pInfo->prefetchedTuple == NULL) {
+ pTupleHandle = tsortNextTuple(pHandle);
+ } else {
+ pTupleHandle = pInfo->prefetchedTuple;
+ pInfo->groupId = tsortGetGroupId(pTupleHandle);
+ pInfo->prefetchedTuple = NULL;
+ }
+ }
+ else {
+ pTupleHandle = tsortNextTuple(pHandle);
+ pInfo->groupId = 0;
+ }
+
if (pTupleHandle == NULL) {
break;
}
- appendOneRowToDataBlock(p, pTupleHandle);
+ if (pInfo->groupSort)
+ {
+ uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+ if (!pInfo->hasGroupId) {
+ pInfo->groupId = tupleGroupId;
+ pInfo->hasGroupId = true;
+ appendOneRowToDataBlock(p, pTupleHandle);
+ } else if (pInfo->groupId == tupleGroupId) {
+ appendOneRowToDataBlock(p, pTupleHandle);
+ } else {
+ pInfo->prefetchedTuple = pTupleHandle;
+ break;
+ }
+ } else {
+ appendOneRowToDataBlock(p, pTupleHandle);
+ }
if (p->info.rows >= capacity) {
break;
}
}
- if (p->info.rows > 0) {// todo extract method
+ if (p->info.rows > 0) { // todo extract method
blockDataEnsureCapacity(pDataBlock, p->info.rows);
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
@@ -593,13 +627,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
-SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
+SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
int32_t code = pOperator->fpSet._openFn(pOperator);
if (code != TSDB_CODE_SUCCESS) {
@@ -608,7 +642,6 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator);
-
if (pBlock != NULL) {
pOperator->resultInfo.totalRows += pBlock->info.rows;
} else {
@@ -617,8 +650,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
return pBlock;
}
-void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
- SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param;
+void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) {
+ SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
@@ -626,11 +659,11 @@ void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pColMatchInfo);
}
-int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
+int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
ASSERT(pOptr != NULL);
SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo));
- SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info;
+ SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info;
*pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle);
*pOptrExplain = pInfo;
@@ -638,24 +671,35 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
return TSDB_CODE_SUCCESS;
}
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
- SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
- SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
- SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
- SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
- int32_t rowSize = pResBlock->info.rowSize;
+SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numStreams,
+ SMergePhysiNode* pMergePhyNode, SExecTaskInfo* pTaskInfo) {
+ SPhysiNode* pPhyNode = (SPhysiNode*)pMergePhyNode;
+
+ SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc;
+ SSDataBlock* pResBlock = createResDataBlock(pDescNode);
+
+ int32_t rowSize = pResBlock->info.rowSize;
if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) {
goto _error;
}
+ SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
+ int32_t numOfOutputCols = 0;
+ SArray* pColMatchColInfo =
+ extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+ SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
+ SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
initResultSizeInfo(pOperator, 1024);
+ pInfo->groupSort = pMergePhyNode->groupSort;
pInfo->binfo.pRes = pResBlock;
pInfo->pSortInfo = pSortInfo;
pInfo->pColMatchInfo = pColMatchColInfo;
pInfo->pInputBlock = pInputBlock;
- pOperator->name = "MultiwaySortMerge";
+ pOperator->name = "MultiwayMerge";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
@@ -667,9 +711,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
// one additional is reserved for merged result.
pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1);
- pOperator->fpSet =
- createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,
- destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo);
+ pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL,
+ destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo);
int32_t code = appendDownstream(pOperator, downStreams, numStreams);
if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 48b0b1c071..963e714972 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -15,6 +15,7 @@
#include "executorimpl.h"
#include "function.h"
#include "functionMgt.h"
+#include "tcompare.h"
#include "tdatablock.h"
#include "tfill.h"
#include "ttime.h"
@@ -26,6 +27,16 @@ typedef enum SResultTsInterpType {
#define IS_FINAL_OP(op) ((op)->isFinal)
+typedef struct SWinRes {
+ TSKEY ts;
+ uint64_t groupId;
+} SWinRes;
+
+typedef struct SPullWindowInfo {
+ STimeWindow window;
+ uint64_t groupId;
+} SPullWindowInfo;
+
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);
@@ -684,11 +695,13 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
}
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
- if (pBlock == NULL) return;
- SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock));
- taosArrayPush(blocks, pBlock);
- blockDebugShowData(blocks, flag);
- taosArrayDestroy(blocks);
+ if (pBlock == NULL){
+ qDebug("======printDataBlock Block is Null");
+ return;
+ }
+ char *pBuf = NULL;
+ qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
+ taosMemoryFree(pBuf);
}
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
@@ -956,6 +969,12 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
getTableScanInfo(pOperator, &pInfo->order, &scanFlag);
+ if (pInfo->scalarSupp.pExprInfo != NULL) {
+ SExprSupp* pExprSup =& pInfo->scalarSupp;
+ projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx,
+ pExprSup->numOfExprs, NULL);
+ }
+
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
@@ -1217,30 +1236,40 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS
}
}
-void doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
+bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
int32_t numOfOutput) {
SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
if (!p1) {
// window has been closed
- return;
+ return false;
}
doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
+ return true;
}
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex,
int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
- SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
- TSKEY* tsCols = (TSKEY*)pColDataInfo->pData;
+ SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex);
+ TSKEY* tsCols = (TSKEY*)pTsCol->pData;
+ uint64_t* pGpDatas = NULL;
+ if (pBlock->info.type == STREAM_RETRIEVE) {
+ SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2);
+ pGpDatas = (uint64_t*)pGpCol->pData;
+ }
int32_t step = 0;
for (int32_t i = 0; i < pBlock->info.rows; i += step) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL);
step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
- doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
- if (pUpWins) {
+ uint64_t groupId = pBlock->info.groupId;
+ if (pGpDatas) {
+ groupId = pGpDatas[i];
+ }
+ bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), groupId, numOfOutput);
+ if (pUpWins && res) {
taosArrayPush(pUpWins, &win);
}
}
@@ -1268,8 +1297,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) {
return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark;
}
-static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
- SArray* closeWins) {
+static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
+ SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) {
void* pIte = NULL;
size_t keyLen = 0;
while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
@@ -1280,10 +1309,20 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
+ SWinRes winRe = {.ts = win.skey, .groupId = groupId,};
+ void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes));
if (isCloseWindow(&win, pSup)) {
- char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
- SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
- taosHashRemove(pHashMap, keyBuf, keyLen);
+ if (chIds && pPullDataMap) {
+ SArray* chAy = *(SArray**) chIds;
+ int32_t size = taosArrayGetSize(chAy);
+ qInfo("======window %ld wait child size:%d", win.skey ,size);
+ for (int32_t i = 0; i < size; i++) {
+ qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i));
+ }
+ continue;
+ } else if (pPullDataMap) {
+ qInfo("======close window %ld", win.skey);
+ }
SResultRowPosition* pPos = (SResultRowPosition*)pIte;
if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
@@ -1291,11 +1330,25 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
return code;
}
}
+ char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
+ SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
+ taosHashRemove(pHashMap, keyBuf, keyLen);
}
}
return TSDB_CODE_SUCCESS;
}
+static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
+ int32_t size = taosArrayGetSize(pChildren);
+ for (int32_t i = 0; i < size; i++) {
+ SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i);
+ SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
+ ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
+ pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
+ closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL);
+ }
+}
+
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
SIntervalAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -1324,7 +1377,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock,
NULL);
qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
@@ -1334,6 +1387,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
continue;
}
+ if (pInfo->scalarSupp.pExprInfo != NULL) {
+ SExprSupp* pExprSup = &pInfo->scalarSupp;
+ projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+ }
+
// The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
// caller. Note that all the time window are not close till now.
// the pDataBlock are always the same one, no need to call this again
@@ -1345,7 +1403,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
}
- closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
+ closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated);
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
@@ -1373,6 +1431,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup);
+ //it should be empty.
+ taosHashCleanup(pInfo->pPullDataMap);
+ taosArrayDestroy(pInfo->pPullWins);
+ blockDataDestroy(pInfo->pPullDataRes);
+
if (pInfo->pChildren) {
int32_t size = taosArrayGetSize(pInfo->pChildren);
for (int32_t i = 0; i < size; i++) {
@@ -1446,18 +1509,27 @@ void increaseTs(SqlFunctionCtx* pCtx) {
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
- STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) {
+ STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) {
SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
- pInfo->win = pTaskInfo->window;
- pInfo->order = TSDB_ORDER_ASC;
- pInfo->interval = *pInterval;
+ pInfo->win = pTaskInfo->window;
+ pInfo->order = TSDB_ORDER_ASC;
+ pInfo->interval = *pInterval;
pInfo->execModel = pTaskInfo->execModel;
- pInfo->twAggSup = *pTwAggSupp;
+ pInfo->twAggSup = *pTwAggSupp;
+
+ if (pPhyNode->window.pExprs != NULL) {
+ int32_t numOfScalar = 0;
+ SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
+ int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+ }
pInfo->primaryTsIndex = primaryTsSlotId;
@@ -2164,6 +2236,24 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
return p1 == NULL;
}
+int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols,
+ int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) {
+ int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos,
+ eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+ int32_t prevEndPos = forwardRows - 1 + startPos;
+ return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
+}
+
+void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) {
+ SArray* childIds = taosArrayInit(8, sizeof(int32_t));
+ for (int32_t i = 0; i < size; i++) {
+ taosArrayPush(childIds, &i);
+ }
+ taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*));
+}
+
+static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
+
static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
SArray* pUpdated) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
@@ -2177,35 +2267,59 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
SResultRow* pResult = NULL;
int32_t forwardRows = 0;
- if (pSDataBlock->pDataBlock != NULL) {
- SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
- tsCols = (int64_t*)pColDataInfo->pData;
- } else {
- return;
- }
+ ASSERT(pSDataBlock->pDataBlock != NULL);
+ SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
+ tsCols = (int64_t*)pColDataInfo->pData;
int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
pInfo->interval.precision, NULL);
while (1) {
- if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
- isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) {
- SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
- taosArrayPush(pUpWins, &nextWin);
- rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs,
- pOperatorInfo->pTaskInfo);
- taosArrayDestroy(pUpWins);
+ if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) {
+ bool ignore = true;
+ SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,};
+ void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
+ if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
+ SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
+ // add pull data request
+ taosArrayPush(pInfo->pPullWins, &pull);
+ addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren));
+ } else {
+ int32_t index = -1;
+ SArray* chArray = NULL;
+ if (chIds) {
+ chArray = *(void**) chIds;
+ int32_t chId = getChildIndex(pSDataBlock);
+ index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
+ }
+ if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) {
+ taosArrayRemove(chArray, index);
+ if (taosArrayGetSize(chArray) == 0) {
+ // pull data is over
+ taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
+ }
+ }
+ if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) {
+ ignore = false;
+ }
+ }
+
+ if (ignore) {
+ startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
+ if (startPos < 0) {
+ break;
+ }
+ continue;
+ }
}
+
int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
- pos->groupId = tableGroupId;
- pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
- *(int64_t*)pos->key = pResult->win.skey;
+
forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
TSDB_ORDER_ASC);
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
@@ -2230,14 +2344,17 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo)
initResultRowInfo(&pInfo->binfo.resultRowInfo);
}
-static void clearUpdateDataBlock(SSDataBlock* pBlock) {
+static void clearSpecialDataBlock(SSDataBlock* pBlock) {
+ if (pBlock->info.rows <= 0) {
+ return;
+ }
blockDataCleanup(pBlock);
}
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
// ASSERT(pDest->info.capacity >= pSource->info.rows);
blockDataEnsureCapacity(pDest, pSource->info.rows);
- clearUpdateDataBlock(pDest);
+ clearSpecialDataBlock(pDest);
SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
@@ -2254,7 +2371,63 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol
blockDataUpdateTsWindow(pDest, 0);
}
-static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
+static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) {
+ int32_t size = taosArrayGetSize(pInfo->pPullWins);
+ if (pInfo->pullIndex < size) {
+ return true;
+ }
+ return false;
+}
+
+static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
+ clearSpecialDataBlock(pBlock);
+ int32_t size = taosArrayGetSize(array);
+ if (size - (*pIndex) == 0) {
+ return;
+ }
+ blockDataEnsureCapacity(pBlock, size - (*pIndex) );
+ ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
+ for (; (*pIndex) < size; (*pIndex)++) {
+ SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) );
+ SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0);
+ colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
+
+ SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1);
+ colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
+
+ SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2);
+ colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
+ pBlock->info.rows++;
+ }
+ if ((*pIndex) == size) {
+ *pIndex = 0;
+ taosArrayClear(array);
+ }
+ blockDataUpdateTsWindow(pBlock, 0);
+}
+
+void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) {
+ SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0);
+ TSKEY* tsData = (TSKEY*)pStartCol->pData;
+ SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2);
+ uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
+ int32_t chId = getChildIndex(pBlock);
+ for (int32_t i = 0; i < pBlock->info.rows; i++) {
+ SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
+ void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
+ if (chIds) {
+ SArray* chArray = *(SArray**) chIds;
+ int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
+ if (index != -1) {
+ taosArrayRemove(chArray, index);
+ if (taosArrayGetSize(chArray) == 0) {
+ // pull data is over
+ taosHashRemove(pMap, &winRes, sizeof(SWinRes));
+ }
+ }
+ }
+ }
+}
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
@@ -2270,35 +2443,57 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
- if (IS_FINAL_OP(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
- if (!IS_FINAL_OP(pInfo)) {
- // semi interval operator clear disk buffer
- clearStreamIntervalOperator(pInfo);
- }
- return NULL;
+ if (!IS_FINAL_OP(pInfo)) {
+ // semi interval operator clear disk buffer
+ clearStreamIntervalOperator(pInfo);
}
+ return NULL;
+ }
+ printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->binfo.pRes;
+ } else {
+ doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+ if (pInfo->binfo.pRes->info.rows != 0) {
+ printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->binfo.pRes;
+ }
+ if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
+ pInfo->returnUpdate = false;
+ ASSERT(!IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
// process the rest of the data
- pOperator->status = OP_OPENED;
return pInfo->pUpdateRes;
}
- return pInfo->binfo.pRes;
+ doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+ if (pInfo->pPullDataRes->info.rows != 0) {
+ // process the rest of the data
+ ASSERT(IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->pPullDataRes;
+ }
}
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
- clearUpdateDataBlock(pInfo->pUpdateRes);
+ clearSpecialDataBlock(pInfo->pUpdateRes);
+ pOperator->status = OP_RES_TO_RETURN;
+ qInfo("Stream Final Interval return data");
break;
}
+ printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv");
+ maxTs = TMAX(maxTs, pBlock->info.window.ekey);
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) {
+ pInfo->binfo.pRes->info.type = pBlock->info.type;
+ } else if (pBlock->info.type == STREAM_CLEAR) {
SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs,
pBlock, pUpWins);
if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
- SIntervalAggOperatorInfo* pChildInfo = pChildOp->info;
+ SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
SExprSupp* pChildSup = &pChildOp->exprSupp;
doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex,
@@ -2310,11 +2505,25 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
removeResults(pUpWins, pUpdated);
copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
+ pInfo->returnUpdate = true;
taosArrayDestroy(pUpWins);
break;
} else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
continue;
+ } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
+ SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
+ doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs,
+ pBlock, pUpWins);
+ removeResults(pUpWins, pUpdated);
+ taosArrayDestroy(pUpWins);
+ if (taosArrayGetSize(pUpdated) > 0) {
+ break;
+ }
+ continue;
+ } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) {
+ processPushEmpty(pBlock, pInfo->pPullDataMap);
+ continue;
}
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
@@ -2334,31 +2543,70 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
- pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, pBlock->info.window.ekey);
+
+ if (needBreak(pInfo)) {
+ break;
+ }
}
- maxTs = TMAX(maxTs, pBlock->info.window.ekey);
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
if (IS_FINAL_OP(pInfo)) {
- closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
+ closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
+ &pInfo->interval, pInfo->pPullDataMap, pUpdated);
+ closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
}
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- pOperator->status = OP_RES_TO_RETURN;
- if (pInfo->binfo.pRes->info.rows == 0) {
- pOperator->status = OP_EXEC_DONE;
- if (pInfo->pUpdateRes->info.rows == 0) {
- return NULL;
- }
+ if (pInfo->binfo.pRes->info.rows != 0) {
+ printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->binfo.pRes;
+ }
+
+ if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
+ pInfo->returnUpdate = false;
+ ASSERT(!IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
// process the rest of the data
- pOperator->status = OP_OPENED;
return pInfo->pUpdateRes;
}
- return pInfo->binfo.pRes;
+
+ doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+ if (pInfo->pPullDataRes->info.rows != 0) {
+ // process the rest of the data
+ ASSERT(IS_FINAL_OP(pInfo));
+ printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+ return pInfo->pPullDataRes;
+ }
+ // ASSERT(false);
+ return NULL;
+}
+
+SSDataBlock* createPullDataBlock() {
+ SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+ pBlock->info.hasVarCol = false;
+ pBlock->info.groupId = 0;
+ pBlock->info.rows = 0;
+ pBlock->info.type = STREAM_RETRIEVE;
+ pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t);
+
+ pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
+ SColumnInfoData infoData = {0};
+ infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
+ infoData.info.bytes = sizeof(TSKEY);
+ // window start ts
+ taosArrayPush(pBlock->pDataBlock, &infoData);
+ // window end ts
+ taosArrayPush(pBlock->pDataBlock, &infoData);
+
+ infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
+ infoData.info.bytes = sizeof(uint64_t);
+ taosArrayPush(pBlock->pDataBlock, &infoData);
+
+ return pBlock;
}
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
@@ -2412,23 +2660,30 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
goto _error;
}
}
- // semi interval operator does not catch result
pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
- pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
+ pInfo->pUpdateRes->info.type = STREAM_CLEAR;
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
+ pInfo->returnUpdate = false;
+
pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
pInfo->isFinal = true;
pOperator->name = "StreamFinalIntervalOperator";
} else {
+ // semi interval operator does not catch result
pInfo->isFinal = false;
pOperator->name = "StreamSemiIntervalOperator";
}
- if (!IS_FINAL_OP(pInfo)) {
+ if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
}
+ pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
+ pInfo->pullIndex = 0;
+ _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+ pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
+ pInfo->pPullDataRes = createPullDataBlock();
pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true;
@@ -2811,11 +3066,6 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
}
}
-typedef struct SWinRes {
- TSKEY ts;
- uint64_t groupId;
-} SWinRes;
-
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
SHashObj* pStDeleted, bool hasEndTs) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -3027,14 +3277,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
} else if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
- /*printDataBlock(pInfo->pDelRes, "session del");*/
+ printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
}
- /*printDataBlock(pBInfo->pRes, "session insert");*/
+ printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@@ -3048,7 +3298,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs,
pInfo->gap, pWins);
@@ -3102,11 +3352,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
- /*printDataBlock(pInfo->pDelRes, "session del");*/
+ printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
- /*printDataBlock(pBInfo->pRes, "session insert");*/
+ printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session");
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
@@ -3169,11 +3419,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
- clearUpdateDataBlock(pInfo->pUpdateRes);
+ clearSpecialDataBlock(pInfo->pUpdateRes);
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins);
removeSessionResults(pStUpdated, pWins);
@@ -3236,7 +3486,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
} else {
pInfo->isFinal = false;
pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
- pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
+ pInfo->pUpdateRes->info.type = STREAM_CLEAR;
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
pOperator->name = "StreamSessionSemiAggOperator";
pOperator->fpSet =
@@ -3554,7 +3804,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
break;
}
- if (pBlock->info.type == STREAM_REPROCESS) {
+ if (pBlock->info.type == STREAM_CLEAR) {
doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
pSeUpdated, pInfo->pSeDeleted);
continue;
@@ -3900,18 +4150,22 @@ _error:
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
SIntervalAggOperatorInfo intervalAggOperatorInfo;
-
- SHashObj* groupIntervalHash;
- void* groupIntervalIter;
+ SList* groupIntervals;
+ SListIter groupIntervalsIter;
bool hasGroupId;
uint64_t groupId;
SSDataBlock* prefetchedBlock;
bool inputBlocksFinished;
} SMergeIntervalAggOperatorInfo;
+typedef struct SGroupTimeWindow {
+ uint64_t groupId;
+ STimeWindow window;
+} SGroupTimeWindow;
+
void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
- taosHashCleanup(miaInfo->groupIntervalHash);
+ tdListFree(miaInfo->groupIntervals);
destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
}
@@ -3940,15 +4194,22 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
bool ascScan = (iaInfo->order == TSDB_ORDER_ASC);
SExprSupp* pExprSup = &pOperatorInfo->exprSupp;
- STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId));
- if (prevWin == NULL) {
- taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow));
- return 0;
- }
+ SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
+ tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
- if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) {
- finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
- taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow));
+ SListIter iter = {0};
+ tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
+ SListNode* listNode = NULL;
+ while ((listNode = tdListNext(&iter)) != NULL) {
+ SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
+ if (prevGrpWin->groupId != tableGroupId ) {
+ continue;
+ }
+ STimeWindow* prevWin = &prevGrpWin->window;
+ if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) {
+ finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
+ tdListPopNode(miaInfo->groupIntervals, listNode);
+ }
}
return 0;
@@ -4075,6 +4336,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
}
if (pBlock == NULL) {
+ tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
miaInfo->inputBlocksFinished = true;
break;
}
@@ -4100,14 +4362,12 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
}
if (miaInfo->inputBlocksFinished) {
- void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter);
- if (win != NULL) {
- miaInfo->groupIntervalIter = win;
+ SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
- size_t len = 0;
- uint64_t* pTableGroupId = taosHashGetKey(win, &len);
- finalizeWindowResult(pOperator, *pTableGroupId, win, pRes);
- pRes->info.groupId = *pTableGroupId;
+ if (listNode != NULL) {
+ SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
+ finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
+ pRes->info.groupId = grpWin->groupId;
}
}
@@ -4129,8 +4389,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI
goto _error;
}
- miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK);
- miaInfo->groupIntervalIter = NULL;
+ miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index f55f7b9ec7..8ee6d18b98 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -758,6 +758,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
}
}
+uint64_t tsortGetGroupId(STupleHandle* pVHandle) {
+ return pVHandle->pBlock->info.groupId;
+}
+
SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
SSortExecInfo info = {0};
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 5c7148932c..d41bc89a5f 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1220,18 +1220,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// The number of parameters has been limited by the syntax definition
- uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+ //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+
// The function return type has been set during syntax parsing
uint8_t para2Type = pFunc->node.resType.type;
- if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
- para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
- para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
- return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
- }
- if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
- (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
- return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
- }
+ //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
+ // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
+ // para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
+ // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ //}
+ //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
+ // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
+ // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ //}
int32_t para2Bytes = pFunc->node.resType.bytes;
if (IS_VAR_DATA_TYPE(para2Type)) {
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index 838071dbf1..983cffe9dc 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -12,6 +12,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+
+// clang-format off
#include "uv.h"
#include "os.h"
#include "fnLog.h"
@@ -25,6 +27,7 @@
#include "tglobal.h"
#include "tmsg.h"
#include "trpc.h"
+// clang-foramt on
typedef struct SUdfdContext {
uv_loop_t * loop;
@@ -103,12 +106,12 @@ typedef struct SUdfdRpcSendRecvInfo {
uv_sem_t resultSem;
} SUdfdRpcSendRecvInfo;
-static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
+static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
static int32_t udfdConnectToMnode();
static int32_t udfdLoadUdf(char *udfName, SUdf *udf);
-static bool udfdRpcRfp(int32_t code);
-static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
+static bool udfdRpcRfp(int32_t code);
+static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static int32_t udfdOpenClientRpc();
static int32_t udfdCloseClientRpc();
@@ -126,19 +129,19 @@ static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn-
static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf);
static void udfdOnNewConnection(uv_stream_t *server, int status);
-static void udfdIntrSignalHandler(uv_signal_t *handle, int signum);
+static void udfdIntrSignalHandler(uv_signal_t *handle, int signum);
static int32_t removeListeningPipe();
-static void udfdPrintVersion();
+static void udfdPrintVersion();
static int32_t udfdParseArgs(int32_t argc, char *argv[]);
static int32_t udfdInitLog();
-static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf);
-static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf);
+static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf);
+static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf);
static int32_t udfdUvInit();
-static void udfdCloseWalkCb(uv_handle_t *handle, void *arg);
+static void udfdCloseWalkCb(uv_handle_t *handle, void *arg);
static int32_t udfdRun();
-static void udfdConnectMnodeThreadFunc(void* args);
+static void udfdConnectMnodeThreadFunc(void *args);
void udfdProcessRequest(uv_work_t *req) {
SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
@@ -401,11 +404,11 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
udf->bufSize = pFuncInfo->bufSize;
char path[PATH_MAX] = {0};
- #ifdef WINDOWS
+#ifdef WINDOWS
snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name);
- #else
+#else
snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name);
- #endif
+#endif
TdFilePtr file =
taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
if (file == NULL) {
@@ -544,7 +547,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
return 0;
}
static bool udfdRpcRfp(int32_t code) {
- if (code == TSDB_CODE_RPC_REDIRECT) {
+ if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
return true;
} else {
return false;
@@ -652,8 +656,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
buf->base = ctx->inputBuf;
buf->len = ctx->inputCap;
} else {
- fnError("udfd can not allocate enough memory")
- buf->base = NULL;
+ fnError("udfd can not allocate enough memory") buf->base = NULL;
buf->len = 0;
}
} else {
@@ -664,8 +667,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
buf->base = ctx->inputBuf + ctx->inputLen;
buf->len = ctx->inputCap - ctx->inputLen;
} else {
- fnError("udfd can not allocate enough memory")
- buf->base = NULL;
+ fnError("udfd can not allocate enough memory") buf->base = NULL;
buf->len = 0;
}
}
@@ -881,7 +883,7 @@ static int32_t udfdRun() {
return 0;
}
-void udfdConnectMnodeThreadFunc(void* args) {
+void udfdConnectMnodeThreadFunc(void *args) {
int32_t retryMnodeTimes = 0;
int32_t code = 0;
while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) {
@@ -939,7 +941,7 @@ int main(int argc, char *argv[]) {
uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL);
udfdRun();
-
+
removeListeningPipe();
udfdCloseClientRpc();
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index ac197912cf..9d8baf472b 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -351,7 +351,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
COPY_SCALAR_FIELD(watermark);
COPY_SCALAR_FIELD(tsColId);
COPY_SCALAR_FIELD(filesFactor);
- CLONE_NODE_LIST_FIELD(pPartTags);
+ CLONE_NODE_LIST_FIELD(pGroupTags);
return TSDB_CODE_SUCCESS;
}
@@ -401,6 +401,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst
CLONE_NODE_LIST_FIELD(pInputs);
COPY_SCALAR_FIELD(numOfChannels);
COPY_SCALAR_FIELD(srcGroupId);
+ COPY_SCALAR_FIELD(groupSort);
return TSDB_CODE_SUCCESS;
}
@@ -436,6 +437,7 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) {
static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pSortKeys);
+ COPY_SCALAR_FIELD(groupSort);
return TSDB_CODE_SUCCESS;
}
@@ -500,7 +502,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy
COPY_SCALAR_FIELD(ratio);
COPY_SCALAR_FIELD(dataRequired);
CLONE_NODE_LIST_FIELD(pDynamicScanFuncs);
- CLONE_NODE_LIST_FIELD(pPartitionTags);
+ CLONE_NODE_LIST_FIELD(pGroupTags);
COPY_SCALAR_FIELD(interval);
COPY_SCALAR_FIELD(offset);
COPY_SCALAR_FIELD(sliding);
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 514d65d52e..df7429bd88 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -234,6 +234,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiMerge";
case QUERY_NODE_PHYSICAL_PLAN_SORT:
return "PhysiSort";
+ case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
+ return "PhysiGroupSort";
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
return "PhysiHashInterval";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@@ -539,7 +541,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
static const char* jkScanLogicPlanTableId = "TableId";
static const char* jkScanLogicPlanTableType = "TableType";
static const char* jkScanLogicPlanTagCond = "TagCond";
-static const char* jkScanLogicPlanPartTags = "PartTags";
+static const char* jkScanLogicPlanGroupTags = "GroupTags";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@@ -561,7 +563,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond);
}
if (TSDB_CODE_SUCCESS == code) {
- code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags);
+ code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags);
}
return code;
@@ -588,7 +590,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond);
}
if (TSDB_CODE_SUCCESS == code) {
- code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags);
+ code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags);
}
return code;
@@ -1430,7 +1432,8 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType";
static const char* jkTableScanPhysiPlanWatermark = "watermark";
static const char* jkTableScanPhysiPlanTsColId = "tsColId";
static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor";
-static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags";
+static const char* jkTableScanPhysiPlanGroupTags = "GroupTags";
+static const char* jkTableScanPhysiPlanGroupSort = "GroupSort";
static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@@ -1485,7 +1488,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor);
}
if (TSDB_CODE_SUCCESS == code) {
- code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags);
+ code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort);
}
return code;
@@ -1544,7 +1550,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor);
}
if (TSDB_CODE_SUCCESS == code) {
- code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags);
+ code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort);
}
return code;
@@ -1725,6 +1734,7 @@ static const char* jkMergePhysiPlanMergeKeys = "MergeKeys";
static const char* jkMergePhysiPlanTargets = "Targets";
static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels";
static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId";
+static const char* jkMergePhysiPlanGroupSort = "GroupSort";
static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj;
@@ -1742,6 +1752,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanGroupSort, pNode->groupSort);
+ }
return code;
}
@@ -1762,6 +1775,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonGetBoolValue(pJson, jkMergePhysiPlanGroupSort, &pNode->groupSort);
+ }
return code;
}
@@ -3369,6 +3385,7 @@ static const char* jkSlotDescSlotId = "SlotId";
static const char* jkSlotDescDataType = "DataType";
static const char* jkSlotDescReserve = "Reserve";
static const char* jkSlotDescOutput = "Output";
+static const char* jkSlotDescName = "Name";
static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
const SSlotDescNode* pNode = (const SSlotDescNode*)pObj;
@@ -3383,6 +3400,9 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonAddStringToObject(pJson, jkSlotDescName, pNode->name);
+ }
return code;
}
@@ -3400,6 +3420,9 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonGetStringValue(pJson, jkSlotDescName, pNode->name);
+ }
return code;
}
@@ -4137,6 +4160,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE:
return physiMergeNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_SORT:
+ case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
return physiSortNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@@ -4280,6 +4304,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE:
return jsonToPhysiMergeNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_SORT:
+ case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
return jsonToPhysiSortNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c
index 1fb2db9f23..3747dde9ed 100644
--- a/source/libs/nodes/src/nodesTraverseFuncs.c
+++ b/source/libs/nodes/src/nodesTraverseFuncs.c
@@ -500,7 +500,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
}
break;
}
- case QUERY_NODE_PHYSICAL_PLAN_SORT: {
+ case QUERY_NODE_PHYSICAL_PLAN_SORT:
+ case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SSortPhysiNode* pSort = (SSortPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 734f287cfb..be6faa92cb 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -288,6 +288,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SMergePhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_SORT:
return makeNode(type, sizeof(SSortPhysiNode));
+ case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
+ return makeNode(type, sizeof(SGroupSortPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
return makeNode(type, sizeof(SIntervalPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@@ -709,7 +711,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pLogicNode->pTagCond);
nodesDestroyNode(pLogicNode->pTagIndexCond);
taosArrayDestroy(pLogicNode->pSmaIndexes);
- nodesDestroyList(pLogicNode->pPartTags);
+ nodesDestroyList(pLogicNode->pGroupTags);
break;
}
case QUERY_NODE_LOGIC_PLAN_JOIN: {
@@ -813,7 +815,7 @@ void nodesDestroyNode(SNode* pNode) {
STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode;
destroyScanPhysiNode((SScanPhysiNode*)pNode);
nodesDestroyList(pPhyNode->pDynamicScanFuncs);
- nodesDestroyList(pPhyNode->pPartitionTags);
+ nodesDestroyList(pPhyNode->pGroupTags);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: {
@@ -850,7 +852,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pPhyNode->pTargets);
break;
}
- case QUERY_NODE_PHYSICAL_PLAN_SORT: {
+ case QUERY_NODE_PHYSICAL_PLAN_SORT:
+ case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pExprs);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index c81ecfc3e9..45240536e3 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -2456,90 +2456,6 @@ static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect)
return cxt.pTranslateCxt->errCode;
}
-typedef struct SRwriteTailCxt {
- STranslateContext* pTranslateCxt;
- int64_t limit;
- int64_t offset;
-} SRwriteTailCxt;
-
-static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) {
- SRwriteTailCxt* pCxt = pContext;
- if (QUERY_NODE_FUNCTION == nodeType(*pNode)) {
- SFunctionNode* pFunc = (SFunctionNode*)*pNode;
- if (FUNCTION_TYPE_TAIL == pFunc->funcType) {
- pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i;
- if (3 == LIST_LENGTH(pFunc->pParameterList)) {
- pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i;
- }
- SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0);
- strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName);
- NODES_CLEAR_LIST(pFunc->pParameterList);
- nodesDestroyNode(*pNode);
- *pNode = pExpr;
- return DEAL_RES_IGNORE_CHILD;
- }
- }
- return DEAL_RES_CONTINUE;
-}
-
-static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) {
- *pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
- if (NULL == *pOutput) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- (*pOutput)->limit = pCxt->limit;
- (*pOutput)->offset = pCxt->offset;
- return TSDB_CODE_SUCCESS;
-}
-
-static SNode* createOrderByExpr(STranslateContext* pCxt) {
- SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
- if (NULL == pOrder) {
- return NULL;
- }
- pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr);
- if (TSDB_CODE_SUCCESS != pCxt->errCode) {
- nodesDestroyNode((SNode*)pOrder);
- return NULL;
- }
- pOrder->order = ORDER_DESC;
- pOrder->nullOrder = NULL_ORDER_FIRST;
- return (SNode*)pOrder;
-}
-
-/* case 1:
- * in: select tail(expr, k, f) from t where_clause
- * out: select expr from t where_clause order by _rowts desc limit k offset f
- *
- * case 2:
- * in: select tail(expr, k, f) from t where_clause partition_by_clause
- * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f
- *
- * case 3:
- * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause
- * out: select expr from (
- * select expr from t where_clause order by _rowts desc limit k offset f
- * ) order_by_clause limit_clause
- *
- * case 4:
- * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause
- * out:
- */
-static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
- if (!pSelect->hasTailFunc) {
- return TSDB_CODE_SUCCESS;
- }
-
- SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1};
- nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt);
- int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt));
- if (TSDB_CODE_SUCCESS == code) {
- code = createLimieNode(&cxt, &pSelect->pLimit);
- }
- pSelect->hasIndefiniteRowsFunc = false;
- return code;
-}
-
typedef struct SReplaceOrderByAliasCxt {
STranslateContext* pTranslateCxt;
SNodeList* pProjectionList;
@@ -2616,9 +2532,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = rewriteUniqueStmt(pCxt, pSelect);
}
- // if (TSDB_CODE_SUCCESS == code) {
- // code = rewriteTailStmt(pCxt, pSelect);
- // }
if (TSDB_CODE_SUCCESS == code) {
code = rewriteTimelineFunc(pCxt, pSelect);
}
@@ -5023,6 +4936,7 @@ typedef struct SVgroupCreateTableBatch {
static void destroyCreateTbReq(SVCreateTbReq* pReq) {
taosMemoryFreeClear(pReq->name);
+ taosMemoryFreeClear(pReq->comment);
taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema);
}
@@ -5040,6 +4954,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
if (pStmt->pOptions->commentNull == false) {
req.comment = strdup(pStmt->pOptions->comment);
if (NULL == req.comment) {
+ destroyCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY;
}
req.commentLen = strlen(pStmt->pOptions->comment);
@@ -5111,6 +5026,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) {
for (int32_t i = 0; i < size; ++i) {
SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i);
taosMemoryFreeClear(pTableReq->name);
+ taosMemoryFreeClear(pTableReq->comment);
if (pTableReq->type == TSDB_NORMAL_TABLE) {
taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema);
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index c4f4624355..202dd96581 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -337,7 +337,11 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
static bool isValidateTag(char* input) {
if (!input) return false;
for (size_t i = 0; i < strlen(input); ++i) {
+ #ifdef WINDOWS
+ if (input[i] < 0x20 || input[i] > 0x7E) return false;
+ #else
if (isprint(input[i]) == 0) return false;
+ #endif
}
return true;
}
@@ -377,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMs
char* jsonKey = item->string;
if (!isValidateTag(jsonKey)) {
+ fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout);
retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
goto end;
}
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index a4cdcd35d3..396a7b6193 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -507,6 +507,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
return TSDB_CODE_OUT_OF_MEMORY;
}
+ pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc;
+
// indefinite rows functions and _select_values functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
@@ -733,6 +735,8 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
return TSDB_CODE_OUT_OF_MEMORY;
}
+ pSort->groupSort = pSelect->groupSort;
+
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets);
if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) {
code = nodesListMakeStrictAppend(&pSort->node.pTargets,
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index c1b12d9d20..53c9ef8dd0 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -20,9 +20,8 @@
#define OPTIMIZE_FLAG_MASK(n) (1 << n)
-#define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0)
-#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1)
-#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2)
+#define OPTIMIZE_FLAG_SCAN_PATH OPTIMIZE_FLAG_MASK(0)
+#define OPTIMIZE_FLAG_PUSH_DOWN_CONDE OPTIMIZE_FLAG_MASK(1)
#define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask)
#define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
@@ -76,7 +75,7 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func)
return NULL;
}
-EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) {
+EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
// *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
*((bool*)pContext) = true;
@@ -85,14 +84,14 @@ EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
-static bool osdHaveNormalCol(SNodeList* pList) {
+static bool scanPathOptHaveNormalCol(SNodeList* pList) {
bool res = false;
- nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res);
+ nodesWalkExprsPostOrder(pList, scanPathOptHaveNormalColImpl, &res);
return res;
}
-static bool osdMayBeOptimized(SLogicNode* pNode) {
- if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OSD)) {
+static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
+ if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH)) {
return false;
}
if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) {
@@ -108,10 +107,10 @@ static bool osdMayBeOptimized(SLogicNode* pNode) {
QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent))) {
return true;
}
- return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
+ return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
}
-static SNodeList* osdGetAllFuncs(SLogicNode* pNode) {
+static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) {
switch (nodeType(pNode)) {
case QUERY_NODE_LOGIC_PLAN_WINDOW:
return ((SWindowLogicNode*)pNode)->pFuncs;
@@ -123,7 +122,7 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) {
return NULL;
}
-static bool needOptimizeDataRequire(const SFunctionNode* pFunc) {
+static bool scanPathOptNeedOptimizeDataRequire(const SFunctionNode* pFunc) {
if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) {
return false;
}
@@ -136,7 +135,7 @@ static bool needOptimizeDataRequire(const SFunctionNode* pFunc) {
return true;
}
-static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) {
+static bool scanPathOptNeedDynOptimize(const SFunctionNode* pFunc) {
if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) {
return false;
}
@@ -149,17 +148,17 @@ static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) {
return true;
}
-static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) {
- SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent);
+static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) {
+ SNodeList* pAllFuncs = scanPathOptGetAllFuncs(pScan->node.pParent);
SNodeList* pTmpSdrFuncs = NULL;
SNodeList* pTmpDsoFuncs = NULL;
SNode* pFunc = NULL;
bool otherFunc = false;
FOREACH(pFunc, pAllFuncs) {
int32_t code = TSDB_CODE_SUCCESS;
- if (needOptimizeDataRequire((SFunctionNode*)pFunc)) {
+ if (scanPathOptNeedOptimizeDataRequire((SFunctionNode*)pFunc)) {
code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pFunc));
- } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) {
+ } else if (scanPathOptNeedDynOptimize((SFunctionNode*)pFunc)) {
code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pFunc));
} else {
otherFunc = true;
@@ -180,15 +179,15 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs,
return TSDB_CODE_SUCCESS;
}
-static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) {
- pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized);
+static int32_t scanPathOptMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) {
+ pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, scanPathOptMayBeOptimized);
if (NULL == pInfo->pScan) {
return TSDB_CODE_SUCCESS;
}
- return osdGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs);
+ return scanPathOptGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs);
}
-static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) {
+static EFuncDataRequired scanPathOptPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) {
switch (l) {
case FUNC_DATA_REQUIRED_DATA_LOAD:
return l;
@@ -202,19 +201,19 @@ static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRe
return r;
}
-static int32_t osdGetDataRequired(SNodeList* pFuncs) {
+static int32_t scanPathOptGetDataRequired(SNodeList* pFuncs) {
if (NULL == pFuncs) {
return FUNC_DATA_REQUIRED_DATA_LOAD;
}
EFuncDataRequired dataRequired = FUNC_DATA_REQUIRED_FILTEROUT;
SNode* pFunc = NULL;
FOREACH(pFunc, pFuncs) {
- dataRequired = osdPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL));
+ dataRequired = scanPathOptPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL));
}
return dataRequired;
}
-static void setScanWindowInfo(SScanLogicNode* pScan) {
+static void scanPathOptSetScanWin(SScanLogicNode* pScan) {
SLogicNode* pParent = pScan->node.pParent;
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && pParent->pParent &&
QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) {
@@ -233,23 +232,23 @@ static void setScanWindowInfo(SScanLogicNode* pScan) {
}
}
-static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SOsdInfo info = {0};
- int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info);
+ int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info);
if (TSDB_CODE_SUCCESS == code && info.pScan) {
- setScanWindowInfo((SScanLogicNode*)info.pScan);
+ scanPathOptSetScanWin((SScanLogicNode*)info.pScan);
}
if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) {
- info.pScan->dataRequired = osdGetDataRequired(info.pSdrFuncs);
+ info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs);
info.pScan->pDynamicScanFuncs = info.pDsoFuncs;
- OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_OSD);
+ OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH);
pCxt->optimized = true;
}
nodesDestroyList(info.pSdrFuncs);
return code;
}
-static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) {
+static int32_t pushDownCondOptMergeCond(SNode** pDst, SNode** pSrc) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -270,7 +269,7 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) {
return code;
}
-static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) {
+static int32_t pushDownCondOptAppendCond(SNode** pCond, SNode** pAdditionalCond) {
if (NULL == *pCond) {
TSWAP(*pCond, *pAdditionalCond);
return TSDB_CODE_SUCCESS;
@@ -283,16 +282,16 @@ static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) {
*pAdditionalCond = NULL;
}
} else {
- code = cpdMergeCond(pCond, pAdditionalCond);
+ code = pushDownCondOptMergeCond(pCond, pAdditionalCond);
}
return code;
}
-static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
- SNode** pOtherCond) {
+static int32_t pushDownCondOptCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
+ SNode** pOtherCond) {
int32_t code = TSDB_CODE_SUCCESS;
if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) {
- code = cpdCondAppend(pOtherCond, pPrimaryKeyCond);
+ code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond);
} else {
bool isStrict = false;
code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict);
@@ -300,7 +299,7 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S
if (isStrict) {
nodesDestroyNode(*pPrimaryKeyCond);
} else {
- code = cpdCondAppend(pOtherCond, pPrimaryKeyCond);
+ code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond);
}
*pPrimaryKeyCond = NULL;
}
@@ -308,8 +307,9 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S
return code;
}
-static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
- if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) ||
+static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
+ if (NULL == pScan->node.pConditions ||
+ OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE) ||
TSDB_SYSTEM_TABLE == pScan->tableType) {
return TSDB_CODE_SUCCESS;
}
@@ -319,14 +319,14 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond,
&pOtherCond);
if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) {
- code = cpdCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond);
+ code = pushDownCondOptCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond);
}
if (TSDB_CODE_SUCCESS == code) {
pScan->node.pConditions = pOtherCond;
}
if (TSDB_CODE_SUCCESS == code) {
- OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD);
+ OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
pCxt->optimized = true;
} else {
nodesDestroyNode(pPrimaryKeyCond);
@@ -336,7 +336,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
return code;
}
-static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
+static bool pushDownCondOptBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
SNode* pTableCol = NULL;
FOREACH(pTableCol, pTableCols) {
if (nodesEqualNode(pCondCol, pTableCol)) {
@@ -346,12 +346,12 @@ static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
return false;
}
-static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) {
+static EDealRes pushDownCondOptIsCrossTableCond(SNode* pNode, void* pContext) {
SCpdIsMultiTableCondCxt* pCxt = pContext;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
- if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) {
+ if (pushDownCondOptBelongThisTable(pNode, pCxt->pLeftCols)) {
pCxt->havaLeftCol = true;
- } else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) {
+ } else if (pushDownCondOptBelongThisTable(pNode, pCxt->pRightCols)) {
pCxt->haveRightCol = true;
}
return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE;
@@ -359,10 +359,11 @@ static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
-static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, SNode* pNode) {
+static ECondAction pushDownCondOptGetCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols,
+ SNode* pNode) {
SCpdIsMultiTableCondCxt cxt = {
.pLeftCols = pLeftCols, .pRightCols = pRightCols, .havaLeftCol = false, .haveRightCol = false};
- nodesWalkExpr(pNode, cpdIsMultiTableCondImpl, &cxt);
+ nodesWalkExpr(pNode, pushDownCondOptIsCrossTableCond, &cxt);
return (JOIN_TYPE_INNER != joinType
? COND_ACTION_STAY
: (cxt.havaLeftCol && cxt.haveRightCol
@@ -370,8 +371,8 @@ static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNode
: (cxt.havaLeftCol ? COND_ACTION_PUSH_LEFT_CHILD : COND_ACTION_PUSH_RIGHT_CHILD)));
}
-static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
- SNode** pRightChildCond) {
+static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
+ SNode** pRightChildCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions;
if (LOGIC_COND_TYPE_AND != pLogicCond->condType) {
return TSDB_CODE_SUCCESS;
@@ -387,7 +388,7 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo
SNodeList* pRemainConds = NULL;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
- ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond);
+ ECondAction condAction = pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond);
if (COND_ACTION_PUSH_JOIN == condAction) {
code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond));
} else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) {
@@ -439,11 +440,12 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo
return code;
}
-static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
- SNode** pRightChildCond) {
+static int32_t pushDownCondOptPartOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
+ SNode** pRightChildCond) {
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
- ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions);
+ ECondAction condAction =
+ pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions);
if (COND_ACTION_STAY == condAction) {
return TSDB_CODE_SUCCESS;
} else if (COND_ACTION_PUSH_JOIN == condAction) {
@@ -457,35 +459,35 @@ static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode*
return TSDB_CODE_SUCCESS;
}
-static int32_t cpdPartitionCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
- SNode** pRightChildCond) {
+static int32_t pushDownCondOptPartCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
+ SNode** pRightChildCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->node.pConditions)) {
- return cpdPartitionLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond);
+ return pushDownCondOptPartLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond);
} else {
- return cpdPartitionOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond);
+ return pushDownCondOptPartOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond);
}
}
-static int32_t cpdPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) {
- return cpdCondAppend(&pJoin->pOnConditions, pCond);
+static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) {
+ return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond);
}
-static int32_t cpdPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) {
- return cpdCondAppend(&pScan->node.pConditions, pCond);
+static int32_t pushDownCondOptPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) {
+ return pushDownCondOptAppendCond(&pScan->node.pConditions, pCond);
}
-static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
+static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
switch (nodeType(pChild)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
- return cpdPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond);
+ return pushDownCondOptPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond);
default:
break;
}
- planError("cpdPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild)));
+ planError("pushDownCondOptPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild)));
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
-static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) {
+static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) {
if (QUERY_NODE_COLUMN != nodeType(pNode)) {
return false;
}
@@ -493,10 +495,10 @@ static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) {
if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) {
return false;
}
- return cpdBelongThisTable(pNode, pTableCols);
+ return pushDownCondOptBelongThisTable(pNode, pTableCols);
}
-static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
+static bool pushDownCondOptIsPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
return false;
}
@@ -508,15 +510,15 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
- if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) {
- return cpdIsPrimaryKey(pOper->pRight, pRightCols);
- } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) {
- return cpdIsPrimaryKey(pOper->pRight, pLeftCols);
+ if (pushDownCondOptIsPriKey(pOper->pLeft, pLeftCols)) {
+ return pushDownCondOptIsPriKey(pOper->pRight, pRightCols);
+ } else if (pushDownCondOptIsPriKey(pOper->pLeft, pRightCols)) {
+ return pushDownCondOptIsPriKey(pOper->pRight, pLeftCols);
}
return false;
}
-static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
+static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond;
if (LOGIC_COND_TYPE_AND != pLogicCond->condType) {
@@ -525,54 +527,56 @@ static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
bool hasPrimaryKeyEqualCond = false;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
- if (cpdContainPrimaryKeyEqualCond(pJoin, pCond)) {
+ if (pushDownCondOptContainPriKeyEqualCond(pJoin, pCond)) {
hasPrimaryKeyEqualCond = true;
break;
}
}
return hasPrimaryKeyEqualCond;
} else {
- return cpdIsPrimaryKeyEqualCond(pJoin, pCond);
+ return pushDownCondOptIsPriKeyEqualCond(pJoin, pCond);
}
}
-static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
+static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN);
}
- if (!cpdContainPrimaryKeyEqualCond(pJoin, pJoin->pOnConditions)) {
+ if (!pushDownCondOptContainPriKeyEqualCond(pJoin, pJoin->pOnConditions)) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
}
return TSDB_CODE_SUCCESS;
}
-static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
- if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
+static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
+ if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
return TSDB_CODE_SUCCESS;
}
if (NULL == pJoin->node.pConditions) {
- return cpdCheckJoinOnCond(pCxt, pJoin);
+ return pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
}
SNode* pOnCond = NULL;
SNode* pLeftChildCond = NULL;
SNode* pRightChildCond = NULL;
- int32_t code = cpdPartitionCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond);
+ int32_t code = pushDownCondOptPartCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond);
if (TSDB_CODE_SUCCESS == code && NULL != pOnCond) {
- code = cpdPushCondToOnCond(pCxt, pJoin, &pOnCond);
+ code = pushDownCondOptPushCondToOnCond(pCxt, pJoin, &pOnCond);
}
if (TSDB_CODE_SUCCESS == code && NULL != pLeftChildCond) {
- code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond);
+ code =
+ pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond);
}
if (TSDB_CODE_SUCCESS == code && NULL != pRightChildCond) {
- code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
+ code =
+ pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
}
if (TSDB_CODE_SUCCESS == code) {
- OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD);
+ OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
pCxt->optimized = true;
- code = cpdCheckJoinOnCond(pCxt, pJoin);
+ code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
} else {
nodesDestroyNode(pOnCond);
nodesDestroyNode(pLeftChildCond);
@@ -582,22 +586,22 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi
return code;
}
-static int32_t cpdPushAggCondition(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
+static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
// todo
return TSDB_CODE_SUCCESS;
}
-static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
+static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pLogicNode)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
- code = cpdOptimizeScanCondition(pCxt, (SScanLogicNode*)pLogicNode);
+ code = pushDownCondOptDealScan(pCxt, (SScanLogicNode*)pLogicNode);
break;
case QUERY_NODE_LOGIC_PLAN_JOIN:
- code = cpdPushJoinCondition(pCxt, (SJoinLogicNode*)pLogicNode);
+ code = pushDownCondOptDealJoin(pCxt, (SJoinLogicNode*)pLogicNode);
break;
case QUERY_NODE_LOGIC_PLAN_AGG:
- code = cpdPushAggCondition(pCxt, (SAggLogicNode*)pLogicNode);
+ code = pushDownCondOptDealAgg(pCxt, (SAggLogicNode*)pLogicNode);
break;
default:
break;
@@ -605,7 +609,7 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode)
if (TSDB_CODE_SUCCESS == code) {
SNode* pChild = NULL;
FOREACH(pChild, pLogicNode->pChildren) {
- code = cpdPushCondition(pCxt, (SLogicNode*)pChild);
+ code = pushDownCondOptimizeImpl(pCxt, (SLogicNode*)pChild);
if (TSDB_CODE_SUCCESS != code) {
break;
}
@@ -614,11 +618,11 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode)
return code;
}
-static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
- return cpdPushCondition(pCxt, pLogicSubplan->pNode);
+static int32_t pushDownCondOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+ return pushDownCondOptimizeImpl(pCxt, pLogicSubplan->pNode);
}
-static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) {
+static bool sortPriKeyOptIsPriKeyOrderBy(SNodeList* pSortKeys) {
if (1 != LIST_LENGTH(pSortKeys)) {
return false;
}
@@ -626,17 +630,18 @@ static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) {
return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false);
}
-static bool opkSortMayBeOptimized(SLogicNode* pNode) {
+static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) {
return false;
}
- if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) {
- return false;
+ SSortLogicNode* pSort = (SSortLogicNode*)pNode;
+ if (pSort->groupSort || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
+ return TSDB_CODE_SUCCESS;
}
return true;
}
-static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) {
+static int32_t sortPriKeyOptGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) {
@@ -646,9 +651,11 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL
}
break;
case QUERY_NODE_LOGIC_PLAN_JOIN:
- code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
+ code =
+ sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
if (TSDB_CODE_SUCCESS == code) {
- code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes);
+ code =
+ sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes);
}
return code;
case QUERY_NODE_LOGIC_PLAN_AGG:
@@ -663,77 +670,58 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL
return TSDB_CODE_SUCCESS;
}
- return opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
+ return sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
}
-static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) {
+static int32_t sortPriKeyOptGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) {
bool notOptimize = false;
- int32_t code = opkGetScanNodesImpl(pNode, ¬Optimize, pScanNodes);
+ int32_t code = sortPriKeyOptGetScanNodesImpl(pNode, ¬Optimize, pScanNodes);
if (TSDB_CODE_SUCCESS != code || notOptimize) {
nodesClearList(*pScanNodes);
}
return code;
}
-static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) {
+static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) {
return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order;
}
-static SNode* opkRewriteDownNode(SSortLogicNode* pSort) {
- SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0);
- // todo
- NODES_CLEAR_LIST(pSort->node.pChildren);
- return pDownNode;
-}
-
-static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) {
- EOrder order = opkGetPrimaryKeyOrder(pSort);
+static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort,
+ SNodeList* pScanNodes) {
+ EOrder order = sortPriKeyOptGetPriKeyOrder(pSort);
if (ORDER_DESC == order) {
SNode* pScan = NULL;
FOREACH(pScan, pScanNodes) { TSWAP(((SScanLogicNode*)pScan)->scanSeq[0], ((SScanLogicNode*)pScan)->scanSeq[1]); }
}
- if (NULL == pSort->node.pParent) {
- // todo
- return TSDB_CODE_SUCCESS;
+ int32_t code =
+ replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0));
+ if (TSDB_CODE_SUCCESS == code) {
+ NODES_CLEAR_LIST(pSort->node.pChildren);
+ nodesDestroyNode((SNode*)pSort);
}
-
- SNode* pDownNode = opkRewriteDownNode(pSort);
- SNode* pNode;
- FOREACH(pNode, pSort->node.pParent->pChildren) {
- if (nodesEqualNode(pNode, (SNode*)pSort)) {
- REPLACE_NODE(pDownNode);
- ((SLogicNode*)pDownNode)->pParent = pSort->node.pParent;
- break;
- }
- }
- nodesDestroyNode((SNode*)pSort);
- return TSDB_CODE_SUCCESS;
+ return code;
}
-static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) {
- OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK);
- if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
- return TSDB_CODE_SUCCESS;
- }
+static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) {
SNodeList* pScanNodes = NULL;
- int32_t code = opkGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes);
+ int32_t code = sortPriKeyOptGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes);
if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) {
- code = opkDoOptimized(pCxt, pSort, pScanNodes);
+ code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pScanNodes);
}
nodesClearList(pScanNodes);
return code;
}
-static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
- SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized);
+static int32_t sortPrimaryKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+ SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortPriKeyOptMayBeOptimized);
if (NULL == pSort) {
return TSDB_CODE_SUCCESS;
}
- return opkOptimizeImpl(pCxt, pSort);
+ return sortPrimaryKeyOptimizeImpl(pCxt, pLogicSubplan, pSort);
}
-static bool smaOptMayBeOptimized(SLogicNode* pNode) {
+static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent ||
QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) ||
WINDOW_TYPE_INTERVAL != ((SWindowLogicNode*)pNode->pParent)->winType) {
@@ -748,7 +736,8 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) {
return true;
}
-static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, SLogicNode** pOutput) {
+static int32_t smaIndexOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets,
+ SLogicNode** pOutput) {
SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
if (NULL == pMerge) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -767,8 +756,8 @@ static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNod
return TSDB_CODE_SUCCESS;
}
-static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge,
- SLogicNode* pSmaScan) {
+static int32_t smaIndexOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge,
+ SLogicNode* pSmaScan) {
int32_t code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pInterval);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pSmaScan);
@@ -781,8 +770,8 @@ static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode*
return code;
}
-static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols,
- SLogicNode** pOutput) {
+static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols,
+ SLogicNode** pOutput) {
SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
if (NULL == pSmaScan) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -811,7 +800,7 @@ static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pInde
return TSDB_CODE_SUCCESS;
}
-static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) {
+static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) {
if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit ||
pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding ||
pWindow->slidingUnit != pIndex->slidingUnit) {
@@ -831,7 +820,7 @@ static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow
return true;
}
-static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) {
+static SNode* smaIndexOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
@@ -846,7 +835,7 @@ static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId)
return (SNode*)pCol;
}
-static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) {
+static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) {
int32_t index = 0;
SNode* pSmaFunc = NULL;
FOREACH(pSmaFunc, pSmaFuncs) {
@@ -858,8 +847,8 @@ static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) {
return -1;
}
-static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput,
- int32_t* pWStrartIndex) {
+static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput,
+ int32_t* pWStrartIndex) {
SNodeList* pCols = NULL;
SNode* pFunc = NULL;
int32_t code = TSDB_CODE_SUCCESS;
@@ -870,11 +859,11 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis
if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) {
*pWStrartIndex = index;
}
- smaFuncIndex = smaOptFindSmaFunc(pFunc, pSmaFuncs);
+ smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs);
if (smaFuncIndex < 0) {
break;
} else {
- code = nodesListMakeStrictAppend(&pCols, smaOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2));
+ code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2));
if (TSDB_CODE_SUCCESS != code) {
break;
}
@@ -891,22 +880,22 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis
return code;
}
-static int32_t smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols,
- int32_t* pWStrartIndex) {
+static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols,
+ int32_t* pWStrartIndex) {
SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent;
- if (!smaOptEqualInterval(pScan, pWindow, pIndex)) {
+ if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) {
return TSDB_CODE_SUCCESS;
}
SNodeList* pSmaFuncs = NULL;
int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs);
if (TSDB_CODE_SUCCESS == code) {
- code = smaOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex);
+ code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex);
}
nodesDestroyList(pSmaFuncs);
return code;
}
-static SNode* smaOptCreateWStartTs() {
+static SNode* smaIndexOptCreateWStartTs() {
SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pWStart) {
return NULL;
@@ -920,7 +909,7 @@ static SNode* smaOptCreateWStartTs() {
return (SNode*)pWStart;
}
-static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) {
+static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) {
SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
if (NULL == pMergeKey) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -935,9 +924,9 @@ static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) {
return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey);
}
-static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) {
+static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) {
if (wstrartIndex < 0) {
- SNode* pWStart = smaOptCreateWStartTs();
+ SNode* pWStart = smaIndexOptCreateWStartTs();
if (NULL == pWStart) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@@ -948,11 +937,11 @@ static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrar
}
wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1;
}
- return smaOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys);
+ return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys);
}
-static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
- SNodeList* pSmaCols, int32_t wstrartIndex) {
+static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
+ SNodeList* pSmaCols, int32_t wstrartIndex) {
SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent;
SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets);
if (NULL == pMergeTargets) {
@@ -961,42 +950,42 @@ static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode*
SLogicNode* pSmaScan = NULL;
SLogicNode* pMerge = NULL;
SNodeList* pMergeKeys = NULL;
- int32_t code = smaOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys);
+ int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
- code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan);
+ code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan);
}
if (TSDB_CODE_SUCCESS == code) {
- code = smaOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge);
+ code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge);
}
if (TSDB_CODE_SUCCESS == code) {
- code = smaOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan);
+ code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan);
}
return code;
}
-static int32_t smaOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
- SNodeList* pSmaCols, int32_t wstrartIndex) {
+static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
+ SNodeList* pSmaCols, int32_t wstrartIndex) {
SLogicNode* pSmaScan = NULL;
- int32_t code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan);
+ int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan);
if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan);
}
return code;
}
-static void smaOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); }
+static void smaIndexOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); }
-static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) {
+static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes);
for (int32_t i = 0; i < nindexes; ++i) {
STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i);
SNodeList* pSmaCols = NULL;
int32_t wstrartIndex = -1;
- code = smaOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex);
+ code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex);
if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) {
- code = smaOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex);
- taosArrayDestroyEx(pScan->pSmaIndexes, smaOptDestroySmaIndex);
+ code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex);
+ taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex);
pScan->pSmaIndexes = NULL;
break;
}
@@ -1004,12 +993,12 @@ static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp
return code;
}
-static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
- SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaOptMayBeOptimized);
+static int32_t smaIndexOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+ SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaIndexOptMayBeOptimized);
if (NULL == pScan) {
return TSDB_CODE_SUCCESS;
}
- return smaOptimizeImpl(pCxt, pLogicSubplan, pScan);
+ return smaIndexOptimizeImpl(pCxt, pLogicSubplan, pScan);
}
static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) {
@@ -1084,7 +1073,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
int32_t code = TSDB_CODE_SUCCESS;
SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0);
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
- TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pPartTags);
+ TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pGroupTags);
int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan);
if (TSDB_CODE_SUCCESS == code) {
NODES_CLEAR_LIST(pNode->pChildren);
@@ -1094,7 +1083,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
SNode* pGroupKey = NULL;
FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) {
code = nodesListMakeStrictAppend(
- &pScan->pPartTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0)));
+ &pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0)));
if (TSDB_CODE_SUCCESS != code) {
break;
}
@@ -1102,11 +1091,29 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys);
}
if (TSDB_CODE_SUCCESS == code) {
- code = partTagsOptRebuildTbanme(pScan->pPartTags);
+ code = partTagsOptRebuildTbanme(pScan->pGroupTags);
}
return code;
}
+static bool eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectNode) {
+ SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ SNode* pProjection;
+ FOREACH(pProjection, pProjectNode->pProjections) {
+ char* projColumnName = ((SColumnNode*)pProjection)->colName;
+ int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName));
+ if (NULL != pExist) {
+ taosHashCleanup(pProjColNameHash);
+ return false;
+ } else {
+ int32_t exist = 1;
+ taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist));
+ }
+ }
+ taosHashCleanup(pProjColNameHash);
+ return true;
+}
+
static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) {
// TODO: enable this optimization after new mechanising that map projection and targets of project node
if (NULL != pNode->pParent) {
@@ -1122,28 +1129,15 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) {
return false;
}
- SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
- SNode* pProjection;
+ SNode* pProjection;
FOREACH(pProjection, pProjectNode->pProjections) {
SExprNode* pExprNode = (SExprNode*)pProjection;
if (QUERY_NODE_COLUMN != nodeType(pExprNode)) {
- taosHashCleanup(pProjColNameHash);
return false;
}
-
- char* projColumnName = ((SColumnNode*)pProjection)->colName;
- int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName));
- if (NULL != pExist) {
- taosHashCleanup(pProjColNameHash);
- return false;
- } else {
- int32_t exist = 1;
- taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist));
- }
}
- taosHashCleanup(pProjColNameHash);
- return true;
+ return eliminateProjOptCheckProjColumnNames(pProjectNode);
}
static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
@@ -1183,14 +1177,199 @@ static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog
return eliminateProjOptimizeImpl(pCxt, pLogicSubplan, pProjectNode);
}
+static bool rewriteTailOptMayBeOptimized(SLogicNode* pNode) {
+ return QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC == nodeType(pNode) && ((SIndefRowsFuncLogicNode*)pNode)->isTailFunc;
+}
+
+static SNode* rewriteTailOptCreateOrderByExpr(SNode* pSortKey) {
+ SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
+ if (NULL == pOrder) {
+ return NULL;
+ }
+ pOrder->order = ORDER_DESC;
+ pOrder->pExpr = nodesCloneNode(pSortKey);
+ if (NULL == pOrder->pExpr) {
+ nodesDestroyNode((SNode*)pOrder);
+ return NULL;
+ }
+ return (SNode*)pOrder;
+}
+
+static int32_t rewriteTailOptCreateLimit(SNode* pLimit, SNode* pOffset, SNode** pOutput) {
+ SLimitNode* pLimitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
+ if (NULL == pLimitNode) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+ pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i;
+ pLimitNode->offset = NULL == pOffset ? -1 : ((SValueNode*)pOffset)->datum.i;
+ *pOutput = (SNode*)pLimitNode;
+ return TSDB_CODE_SUCCESS;
+}
+
+static bool rewriteTailOptNeedGroupSort(SIndefRowsFuncLogicNode* pIndef) {
+ if (1 != LIST_LENGTH(pIndef->node.pChildren)) {
+ return false;
+ }
+ SNode* pChild = nodesListGetNode(pIndef->node.pChildren, 0);
+ return QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pChild) ||
+ (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && NULL != ((SScanLogicNode*)pChild)->pGroupTags);
+}
+
+static int32_t rewriteTailOptCreateSort(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) {
+ SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT);
+ if (NULL == pSort) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ pSort->groupSort = rewriteTailOptNeedGroupSort(pIndef);
+ TSWAP(pSort->node.pChildren, pIndef->node.pChildren);
+ pSort->node.precision = pIndef->node.precision;
+
+ // tail(expr, [limit, offset,] _rowts)
+ SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0);
+ int32_t rowtsIndex = LIST_LENGTH(pTail->pParameterList) - 1;
+
+ int32_t code = nodesListMakeStrictAppend(
+ &pSort->pSortKeys, rewriteTailOptCreateOrderByExpr(nodesListGetNode(pTail->pParameterList, rowtsIndex)));
+ if (TSDB_CODE_SUCCESS == code) {
+ pSort->node.pTargets = nodesCloneList(((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0))->pTargets);
+ if (NULL == pSort->node.pTargets) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ }
+ }
+
+ if (TSDB_CODE_SUCCESS == code) {
+ *pOutput = (SLogicNode*)pSort;
+ } else {
+ nodesDestroyNode((SNode*)pSort);
+ }
+
+ return code;
+}
+
+static SNode* rewriteTailOptCreateProjectExpr(SFunctionNode* pTail) {
+ SNode* pExpr = nodesCloneNode(nodesListGetNode(pTail->pParameterList, 0));
+ if (NULL == pExpr) {
+ return NULL;
+ }
+ strcpy(((SExprNode*)pExpr)->aliasName, pTail->node.aliasName);
+ return pExpr;
+}
+
+static int32_t rewriteTailOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) {
+ SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT);
+ if (NULL == pProject) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ TSWAP(pProject->node.pTargets, pIndef->node.pTargets);
+ pProject->node.precision = pIndef->node.precision;
+
+ // tail(expr, [limit, offset,] _rowts)
+ SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0);
+ int32_t limitIndex = LIST_LENGTH(pTail->pParameterList) > 2 ? 1 : -1;
+ int32_t offsetIndex = LIST_LENGTH(pTail->pParameterList) > 3 ? 2 : -1;
+
+ int32_t code = nodesListMakeStrictAppend(&pProject->pProjections, rewriteTailOptCreateProjectExpr(pTail));
+ if (TSDB_CODE_SUCCESS == code) {
+ code = rewriteTailOptCreateLimit(limitIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, limitIndex),
+ offsetIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, offsetIndex),
+ &pProject->node.pLimit);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ *pOutput = (SLogicNode*)pProject;
+ } else {
+ nodesDestroyNode((SNode*)pProject);
+ }
+ return code;
+}
+
+static int32_t rewriteTailOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
+ SIndefRowsFuncLogicNode* pIndef) {
+ SLogicNode* pSort = NULL;
+ SLogicNode* pProject = NULL;
+ int32_t code = rewriteTailOptCreateSort(pIndef, &pSort);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = rewriteTailOptCreateProject(pIndef, &pProject);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pSort);
+ pSort->pParent = pProject;
+ pSort = NULL;
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ nodesDestroyNode((SNode*)pIndef);
+ } else {
+ nodesDestroyNode((SNode*)pSort);
+ nodesDestroyNode((SNode*)pProject);
+ }
+ return code;
+}
+
+static int32_t rewriteTailOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+ SIndefRowsFuncLogicNode* pIndef =
+ (SIndefRowsFuncLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, rewriteTailOptMayBeOptimized);
+
+ if (NULL == pIndef) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ return rewriteTailOptimizeImpl(pCxt, pLogicSubplan, pIndef);
+}
+
+static bool eliminateSetOpMayBeOptimized(SLogicNode* pNode) {
+ SLogicNode* pParent = pNode->pParent;
+ if (NULL == pParent ||
+ QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) ||
+ LIST_LENGTH(pParent->pChildren) < 2) {
+ return false;
+ }
+ if (nodeType(pNode) != nodeType(pNode->pParent) || LIST_LENGTH(pNode->pChildren) < 2) {
+ return false;
+ }
+ return true;
+}
+
+static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
+ SLogicNode* pSetOpNode) {
+ SNode* pSibling;
+ FOREACH(pSibling, pSetOpNode->pParent->pChildren) {
+ if (nodesEqualNode(pSibling, (SNode*)pSetOpNode)) {
+ SNode* pChild;
+ FOREACH(pChild, pSetOpNode->pChildren) { ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; }
+ INSERT_LIST(pSetOpNode->pParent->pChildren, pSetOpNode->pChildren);
+
+ pSetOpNode->pChildren = NULL;
+ ERASE_NODE(pSetOpNode->pParent->pChildren);
+ return TSDB_CODE_SUCCESS;
+ }
+ }
+
+ return TSDB_CODE_PLAN_INTERNAL_ERROR;
+}
+
+static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+ SLogicNode* pSetOpNode = optFindPossibleNode(pLogicSubplan->pNode, eliminateSetOpMayBeOptimized);
+ if (NULL == pSetOpNode) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ return eliminateSetOpOptimizeImpl(pCxt, pLogicSubplan, pSetOpNode);
+}
+
// clang-format off
static const SOptimizeRule optimizeRuleSet[] = {
- {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize},
- {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize},
- {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize},
- {.pName = "SmaIndex", .optimizeFunc = smaOptimize},
- {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
- {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}
+ {.pName = "ScanPath", .optimizeFunc = scanPathOptimize},
+ {.pName = "PushDownCondition", .optimizeFunc = pushDownCondOptimize},
+ {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize},
+ {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize},
+ {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
+ {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
+ {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
+ {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}
};
// clang-format on
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index 4f46abd8da..46747af3a9 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -41,8 +41,12 @@ typedef struct SPhysiPlanContext {
static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
- if (NULL != pStmtName && '\0' != pStmtName[0]) {
- return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName);
+ if (NULL != pStmtName) {
+ if ('\0' != pStmtName[0]) {
+ return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName);
+ } else {
+ return sprintf(pKey, "%s", pCol->node.aliasName);
+ }
}
if ('\0' == pCol->tableAlias[0]) {
return sprintf(pKey, "%s", pCol->colName);
@@ -56,11 +60,13 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
}
-static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output, bool reserve) {
+static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId,
+ bool output, bool reserve) {
SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC);
if (NULL == pSlot) {
return NULL;
}
+ strcpy(pSlot->name, pName);
pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->reserve = reserve;
@@ -99,10 +105,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char
return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex));
}
-static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
- char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
- int32_t len = getSlotKey(pNode, NULL, name);
- return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash);
+static int32_t putSlotToHash(const char* pName, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
+ return putSlotToHashImpl(dataBlockId, slotId, pName, strlen(pName), pHash);
}
static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId,
@@ -131,9 +135,11 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
int16_t slotId = 0;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
- code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true, false));
+ char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
+ getSlotKey(pNode, NULL, name);
+ code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false));
if (TSDB_CODE_SUCCESS == code) {
- code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
+ code = putSlotToHash(name, pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
}
if (TSDB_CODE_SUCCESS == code) {
pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes;
@@ -196,7 +202,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
int32_t len = getSlotKey(pExpr, pStmtName, name);
SSlotIndex* pIndex = taosHashGet(pHash, name, len);
if (NULL == pIndex) {
- code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output, reserve));
+ code =
+ nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve));
if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
}
@@ -513,12 +520,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
pTableScan->dataRequired = pScanLogicNode->dataRequired;
pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs);
- pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags);
+ pTableScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags);
if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) ||
- (NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) {
+ (NULL != pScanLogicNode->pGroupTags && NULL == pTableScan->pGroupTags)) {
nodesDestroyNode((SNode*)pTableScan);
return TSDB_CODE_OUT_OF_MEMORY;
}
+ pTableScan->groupSort = pScanLogicNode->groupSort;
pTableScan->interval = pScanLogicNode->interval;
pTableScan->offset = pScanLogicNode->offset;
pTableScan->sliding = pScanLogicNode->sliding;
@@ -1170,8 +1178,9 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr
static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode,
SPhysiNode** pPhyNode) {
- SSortPhysiNode* pSort =
- (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT);
+ SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(
+ pCxt, (SLogicNode*)pSortLogicNode,
+ pSortLogicNode->groupSort ? QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT : QUERY_NODE_PHYSICAL_PLAN_SORT);
if (NULL == pSort) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@@ -1185,7 +1194,7 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs);
if (TSDB_CODE_SUCCESS == code) {
- code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe);
+ code = pushdownDataBlockSlots(pCxt, pSort->pExprs, pChildTupe);
}
}
@@ -1322,6 +1331,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
pMerge->numOfChannels = pMergeLogicNode->numOfChannels;
pMerge->srcGroupId = pMergeLogicNode->srcGroupId;
+ pMerge->groupSort = pMergeLogicNode->groupSort;
int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc);
diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index b4a966f206..49ab50f913 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -362,7 +362,7 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) {
}
static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode,
- SNodeList* pMergeKeys, SLogicNode* pPartChild) {
+ SNodeList* pMergeKeys, SLogicNode* pPartChild, bool groupSort) {
SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
if (NULL == pMerge) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -371,6 +371,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla
pMerge->srcGroupId = pCxt->groupId;
pMerge->node.precision = pPartChild->precision;
pMerge->pMergeKeys = pMergeKeys;
+ pMerge->groupSort = groupSort;
int32_t code = TSDB_CODE_SUCCESS;
pMerge->pInputs = nodesCloneList(pPartChild->pTargets);
@@ -430,7 +431,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo
SNodeList* pMergeKeys = NULL;
code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
- code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow);
+ code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pMergeKeys);
@@ -497,12 +498,16 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
return code;
}
-static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) {
+static void stbSplSetTableMergeScan(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
- ((SScanLogicNode*)pNode)->scanType = scanType;
+ SScanLogicNode* pScan = (SScanLogicNode*)pNode;
+ pScan->scanType = SCAN_TYPE_TABLE_MERGE;
+ if (NULL != pScan->pGroupTags) {
+ pScan->groupSort = true;
+ }
} else {
if (1 == LIST_LENGTH(pNode->pChildren)) {
- splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType);
+ stbSplSetTableMergeScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
}
}
}
@@ -515,7 +520,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl
int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
- code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild);
+ code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true);
}
if (TSDB_CODE_SUCCESS == code) {
@@ -524,13 +529,10 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl
}
if (TSDB_CODE_SUCCESS == code) {
- splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE);
- ++(pCxt->groupId);
- }
-
- if (TSDB_CODE_SUCCESS == code) {
+ stbSplSetTableMergeScan(pChild);
pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
+ ++(pCxt->groupId);
} else {
nodesDestroyList(pMergeKeys);
}
@@ -560,7 +562,7 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
- return ((SScanLogicNode*)pNode)->pPartTags;
+ return ((SScanLogicNode*)pNode)->pGroupTags;
} else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
return ((SPartitionLogicNode*)pNode)->pPartitionKeys;
} else {
@@ -775,6 +777,7 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut
pPartSort->node.pChildren = pChildren;
splSetParent((SLogicNode*)pPartSort);
pPartSort->pSortKeys = pSortKeys;
+ pPartSort->groupSort = pSort->groupSort;
code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys);
}
@@ -789,12 +792,29 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut
return code;
}
+static void stbSplSetScanPartSort(SLogicNode* pNode) {
+ if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
+ SScanLogicNode* pScan = (SScanLogicNode*)pNode;
+ if (NULL != pScan->pGroupTags) {
+ pScan->groupSort = true;
+ }
+ } else {
+ if (1 == LIST_LENGTH(pNode->pChildren)) {
+ stbSplSetScanPartSort((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
+ }
+ }
+}
+
static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
SLogicNode* pPartSort = NULL;
SNodeList* pMergeKeys = NULL;
+ bool groupSort = ((SSortLogicNode*)pInfo->pSplitNode)->groupSort;
int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
- code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort);
+ code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort, groupSort);
+ }
+ if (TSDB_CODE_SUCCESS == code && groupSort) {
+ stbSplSetScanPartSort(pPartSort);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
@@ -829,7 +849,7 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS
SNodeList* pMergeKeys = NULL;
int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
- code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan);
+ code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pSubplan->pChildren,
diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp
index ea40567dae..8c19b52a09 100644
--- a/source/libs/planner/test/planBasicTest.cpp
+++ b/source/libs/planner/test/planBasicTest.cpp
@@ -67,6 +67,14 @@ TEST_F(PlanBasicTest, tailFunc) {
run("SELECT TAIL(c1, 10) FROM t1");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10");
+
+ run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1");
+
+ run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 ORDER BY 1");
+
+ run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5");
+
+ run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5");
}
TEST_F(PlanBasicTest, interpFunc) {
diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp
index 43b6bf5772..46c2f33048 100644
--- a/source/libs/planner/test/planTestMain.cpp
+++ b/source/libs/planner/test/planTestMain.cpp
@@ -76,6 +76,7 @@ static void parseArg(int argc, char* argv[]) {
static struct option long_options[] = {
{"dump", optional_argument, NULL, 'd'},
{"skipSql", required_argument, NULL, 's'},
+ {"limitSql", required_argument, NULL, 'i'},
{"log", required_argument, NULL, 'l'},
{0, 0, 0, 0}
};
@@ -88,6 +89,9 @@ static void parseArg(int argc, char* argv[]) {
case 's':
setSkipSqlNum(optarg);
break;
+ case 'i':
+ setLimitSqlNum(optarg);
+ break;
case 'l':
setLogLevel(optarg);
break;
diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp
index c197a02dd1..d54859c296 100644
--- a/source/libs/planner/test/planTestUtil.cpp
+++ b/source/libs/planner/test/planTestUtil.cpp
@@ -51,6 +51,7 @@ enum DumpModule {
DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
int32_t g_skipSql = 0;
+int32_t g_limitSql = 0;
int32_t g_logLevel = 131;
void setDumpModule(const char* pModule) {
@@ -76,28 +77,33 @@ void setDumpModule(const char* pModule) {
}
void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); }
-
+void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); }
void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); }
int32_t getLogLevel() { return g_logLevel; }
class PlannerTestBaseImpl {
public:
- PlannerTestBaseImpl() : sqlNo_(0) {}
+ PlannerTestBaseImpl() : sqlNo_(0), sqlNum_(0) {}
void useDb(const string& user, const string& db) {
caseEnv_.acctId_ = 0;
caseEnv_.user_ = user;
caseEnv_.db_ = db;
- caseEnv_.nsql_ = g_skipSql;
+ caseEnv_.numOfSkipSql_ = g_skipSql;
+ caseEnv_.numOfLimitSql_ = g_limitSql;
}
void run(const string& sql) {
++sqlNo_;
- if (caseEnv_.nsql_ > 0) {
- --(caseEnv_.nsql_);
+ if (caseEnv_.numOfSkipSql_ > 0) {
+ --(caseEnv_.numOfSkipSql_);
return;
}
+ if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) {
+ return;
+ }
+ ++sqlNum_;
reset();
try {
@@ -134,7 +140,7 @@ class PlannerTestBaseImpl {
}
void prepare(const string& sql) {
- if (caseEnv_.nsql_ > 0) {
+ if (caseEnv_.numOfSkipSql_ > 0) {
return;
}
@@ -148,7 +154,7 @@ class PlannerTestBaseImpl {
}
void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) {
- if (caseEnv_.nsql_ > 0) {
+ if (caseEnv_.numOfSkipSql_ > 0) {
return;
}
@@ -161,8 +167,8 @@ class PlannerTestBaseImpl {
}
void exec() {
- if (caseEnv_.nsql_ > 0) {
- --(caseEnv_.nsql_);
+ if (caseEnv_.numOfSkipSql_ > 0) {
+ --(caseEnv_.numOfSkipSql_);
return;
}
@@ -197,9 +203,10 @@ class PlannerTestBaseImpl {
int32_t acctId_;
string user_;
string db_;
- int32_t nsql_;
+ int32_t numOfSkipSql_;
+ int32_t numOfLimitSql_;
- caseEnv() : nsql_(0) {}
+ caseEnv() : numOfSkipSql_(0) {}
};
struct stmtEnv {
@@ -401,6 +408,7 @@ class PlannerTestBaseImpl {
stmtEnv stmtEnv_;
stmtRes res_;
int32_t sqlNo_;
+ int32_t sqlNum_;
};
PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {}
diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h
index 7d2a9e533f..f9942c93a7 100644
--- a/source/libs/planner/test/planTestUtil.h
+++ b/source/libs/planner/test/planTestUtil.h
@@ -43,6 +43,7 @@ class PlannerTestBase : public testing::Test {
extern void setDumpModule(const char* pModule);
extern void setSkipSqlNum(const char* pNum);
+extern void setLimitSqlNum(const char* pNum);
extern void setLogLevel(const char* pLogLevel);
extern int32_t getLogLevel();
diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index c807c1c4cd..bee0461e58 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -19,6 +19,7 @@
#include "tmsg.h"
#include "trpc.h"
#include "tsched.h"
+// clang-format off
#include "cJSON.h"
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
@@ -147,13 +148,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra
}
memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len);
- SRpcMsg rpcMsg = {.msgType = pInfo->msgType,
- .pCont = pMsg,
- .contLen = pInfo->msgInfo.len,
- .info.ahandle = (void*)pInfo,
- .info.handle = pInfo->msgInfo.handle,
- .info.persistHandle = persistHandle,
- .code = 0};
+ SRpcMsg rpcMsg = {
+ .msgType = pInfo->msgType,
+ .pCont = pMsg,
+ .contLen = pInfo->msgInfo.len,
+ .info.ahandle = (void*)pInfo,
+ .info.handle = pInfo->msgInfo.handle,
+ .info.persistHandle = persistHandle,
+ .code = 0
+ };
assert(pInfo->fp != NULL);
TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId);
rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx);
@@ -221,8 +224,9 @@ void destroyQueryExecRes(SQueryExecRes* pRes) {
qError("invalid exec result for request type %d", pRes->msgType);
}
}
+// clang-format on
-int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
+int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) {
int32_t n = 0;
switch (type) {
@@ -262,7 +266,7 @@ int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (bufSize < 0) {
-// tscError("invalid buf size");
+ // tscError("invalid buf size");
return TSDB_CODE_TSC_INVALID_VALUE;
}
@@ -289,7 +293,7 @@ int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
break;
default:
-// tscError("unsupported type:%d", type);
+ // tscError("unsupported type:%d", type);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@@ -332,7 +336,7 @@ char* parseTagDatatoJson(void* p) {
int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue);
if (length < 0) {
qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
- pTagVal->pData);
+ pTagVal->pData);
taosMemoryFree(tagJsonValue);
goto end;
}
@@ -372,7 +376,6 @@ end:
return string;
}
-
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
if (NULL == pSrc) {
*pDst = NULL;
@@ -393,37 +396,36 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) {
*pDst = NULL;
return TSDB_CODE_SUCCESS;
}
-
+
*pDst = taosMemoryMalloc(sizeof(*pSrc));
if (NULL == *pDst) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy(*pDst, pSrc, sizeof(*pSrc));
if (pSrc->vgHash) {
- (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+ (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true,
+ HASH_ENTRY_LOCK);
if (NULL == (*pDst)->vgHash) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SVgroupInfo* vgInfo = NULL;
- void *pIter = taosHashIterate(pSrc->vgHash, NULL);
+ void* pIter = taosHashIterate(pSrc->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
int32_t* vgId = taosHashGetKey(pIter, NULL);
-
+
if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) {
qError("taosHashPut failed, vgId:%d", vgInfo->vgId);
- taosHashCancelIterate(pSrc->vgHash, pIter);
+ taosHashCancelIterate(pSrc->vgHash, pIter);
taosHashCleanup((*pDst)->vgHash);
taosMemoryFreeClear(*pDst);
return TSDB_CODE_CTG_MEM_ERROR;
}
-
+
pIter = taosHashIterate(pSrc->vgHash, pIter);
}
}
-
+
return TSDB_CODE_SUCCESS;
}
-
-
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index d7f0733129..80bebafef2 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -735,6 +735,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
char *input = colDataGetData(pInput[0].columnData, i);
switch(outputType) {
+ case TSDB_DATA_TYPE_TINYINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_SMALLINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_INT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input);
+ }
+ break;
+ }
case TSDB_DATA_TYPE_BIGINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10);
@@ -753,6 +807,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
break;
}
+ case TSDB_DATA_TYPE_UTINYINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_USMALLINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_UINT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input);
+ }
+ break;
+ }
case TSDB_DATA_TYPE_UBIGINT: {
if (inputType == TSDB_DATA_TYPE_BINARY) {
*(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10);
@@ -771,10 +879,64 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
break;
}
+ case TSDB_DATA_TYPE_FLOAT: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(float *)output = taosStr2Float(varDataVal(input), NULL);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(float *)output = taosStr2Float(newBuf, NULL);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(float *)output, float, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_DOUBLE: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(double *)output = taosStr2Double(varDataVal(input), NULL);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(double *)output = taosStr2Double(newBuf, NULL);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(double *)output, double, inputType, input);
+ }
+ break;
+ }
+ case TSDB_DATA_TYPE_BOOL: {
+ if (inputType == TSDB_DATA_TYPE_BINARY) {
+ *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10);
+ } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ newBuf[len] = 0;
+ *(bool *)output = taosStr2Int8(newBuf, NULL, 10);
+ taosMemoryFree(newBuf);
+ } else {
+ GET_TYPED_DATA(*(bool *)output, bool, inputType, input);
+ }
+ break;
+ }
case TSDB_DATA_TYPE_TIMESTAMP: {
if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
- //not support
- return TSDB_CODE_FAILED;
+ //convert to 0
+ *(int64_t *)output = 0;
} else {
GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
}
@@ -789,8 +951,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input));
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
- //not support
- return TSDB_CODE_FAILED;
+ char *newBuf = taosMemoryCalloc(1, inputLen);
+ int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+ if (len < 0) {
+ taosMemoryFree(newBuf);
+ return TSDB_CODE_FAILED;
+ }
+ len = TMIN(len, outputLen - VARSTR_HEADER_SIZE);
+ memcpy(varDataVal(output), newBuf, len);
+ varDataSetLen(output, len);
+ taosMemoryFree(newBuf);
} else {
char tmp[400] = {0};
NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index d619615144..18bbacd5b7 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -477,7 +477,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
}
void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar
- int32_t inputLen = varDataLen(buf);
+ int32_t inputLen = varDataTLen(buf);
void* t = taosMemoryCalloc(1, inputLen);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t));
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 9d92865e47..56d063ae51 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -111,7 +111,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
// enqueue
if (pData != NULL) {
- pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+ pData->type = STREAM_INPUT__DATA_BLOCK;
pData->srcVgId = pReq->dataSrcVgId;
// decode
/*pData->blocks = pReq->data;*/
@@ -146,7 +146,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
// enqueue
if (pData != NULL) {
- pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+ pData->type = STREAM_INPUT__DATA_RETRIEVE;
pData->srcVgId = 0;
// decode
/*pData->blocks = pReq->data;*/
@@ -170,7 +170,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
pCont->rspToTaskId = pReq->srcTaskId;
pCont->rspFromTaskId = pReq->dstTaskId;
pRsp->pCont = buf;
- pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
+ pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp);
tmsgSendRsp(pRsp);
return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
}
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 1f51a927e7..a5e9b8edd9 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -115,6 +115,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
.srcNodeId = pTask->nodeId,
.srcTaskId = pTask->taskId,
.pRetrieve = pRetrieve,
+ .retrieveLen = dataStrLen,
};
int32_t sz = taosArrayGetSize(pTask->childEpInfo);
@@ -146,7 +147,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
.code = 0,
.msgType = TDMT_STREAM_RETRIEVE,
.pCont = buf,
- .contLen = len,
+ .contLen = sizeof(SMsgHead) + len,
};
if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
@@ -300,7 +301,7 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) {
atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
return 0;
}
- ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK);
+ ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
qInfo("stream continue dispatching: task %d", pTask->taskId);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index cb7dd241c1..c75e6c004a 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -17,6 +17,7 @@
static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
void* exec = pTask->exec.executor;
+ bool hasData = false;
// set input
SStreamQueueItem* pItem = (SStreamQueueItem*)data;
@@ -27,7 +28,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
ASSERT(pTask->isDataScan);
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
- } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+ } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
SArray* blocks = pBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
@@ -43,7 +44,21 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
- if (output == NULL) break;
+ if (output == NULL) {
+ if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
+ //SSDataBlock block = {0};
+ //block.info.type = STREAM_PUSH_EMPTY;
+ //block.info.childId = pTask->selfChildId;
+ SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data;
+ ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
+ SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);
+ pBlock->info.type = STREAM_PUSH_EMPTY;
+ pBlock->info.childId = pTask->selfChildId;
+ taosArrayPush(pRes, pBlock);
+ }
+ break;
+ }
+ hasData = true;
if (output->info.type == STREAM_RETRIEVE) {
if (streamBroadcastToChildren(pTask, output) < 0) {
@@ -99,7 +114,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
if (type == STREAM_INPUT__TRIGGER) {
blockDataDestroy(((SStreamTrigger*)data)->pBlock);
taosFreeQitem(data);
- } else if (type == STREAM_INPUT__DATA_BLOCK) {
+ } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE) {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c
index ada391b40a..3d64cec8d8 100644
--- a/source/libs/stream/src/streamUpdate.c
+++ b/source/libs/stream/src/streamUpdate.c
@@ -119,6 +119,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma
taosArrayPush(pInfo->pTsBuckets, &dumy);
}
pInfo->numBuckets = DEFAULT_BUCKET_SIZE;
+ pInfo->pCloseWinSBF = NULL;
return pInfo;
}
@@ -154,6 +155,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) {
TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index);
if (ts < maxTs - pInfo->watermark) {
// this window has been closed.
+ if (pInfo->pCloseWinSBF) {
+ return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY));
+ }
return true;
}
@@ -193,3 +197,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
taosArrayDestroy(pInfo->pTsSBFs);
taosMemoryFree(pInfo);
}
+
+void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) {
+ if (pInfo->pCloseWinSBF) {
+ return;
+ }
+ int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
+ pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
+}
+
+void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) {
+ if (!pInfo || !pInfo->pCloseWinSBF) {
+ return;
+ }
+ tScalableBfDestroy(pInfo->pCloseWinSBF);
+ pInfo->pCloseWinSBF = NULL;
+}
diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h
index 2e8e0e1227..beddec64c5 100644
--- a/source/libs/sync/inc/syncEnv.h
+++ b/source/libs/sync/inc/syncEnv.h
@@ -28,13 +28,13 @@ extern "C" {
#include "trpc.h"
#include "ttimer.h"
-#define TIMER_MAX_MS 0x7FFFFFFF
-#define ENV_TICK_TIMER_MS 1000
-#define PING_TIMER_MS 1000
-#define ELECT_TIMER_MS_MIN 1300
-#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
+#define TIMER_MAX_MS 0x7FFFFFFF
+#define ENV_TICK_TIMER_MS 1000
+#define PING_TIMER_MS 1000
+#define ELECT_TIMER_MS_MIN 1300
+#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
-#define HEARTBEAT_TIMER_MS 900
+#define HEARTBEAT_TIMER_MS 900
#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})
diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h
index 88d3065375..f3064e668d 100644
--- a/source/libs/sync/inc/syncIO.h
+++ b/source/libs/sync/inc/syncIO.h
@@ -50,7 +50,7 @@ typedef struct SSyncIO {
void *pSyncNode;
int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg);
int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg);
- int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg);
+ int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg, SyncIndex *pRetIndex);
int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg);
int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg);
int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg);
diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 1bd0671fb5..d351dc50f4 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -169,7 +169,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
void syncNodeStart(SSyncNode* pSyncNode);
void syncNodeStartStandBy(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode);
-int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak);
+int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak);
// option
bool syncNodeSnapshotEnable(SSyncNode* pSyncNode);
@@ -233,6 +233,7 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index);
SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index);
int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm);
+bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg);
int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag);
int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg);
diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h
index efe8a65b77..521ca6068d 100644
--- a/source/libs/sync/inc/syncRaftCfg.h
+++ b/source/libs/sync/inc/syncRaftCfg.h
@@ -49,14 +49,14 @@ int32_t raftCfgClose(SRaftCfg *pRaftCfg);
int32_t raftCfgPersist(SRaftCfg *pRaftCfg);
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex);
-cJSON *syncCfg2Json(SSyncCfg *pSyncCfg);
-char *syncCfg2Str(SSyncCfg *pSyncCfg);
-char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg);
+cJSON * syncCfg2Json(SSyncCfg *pSyncCfg);
+char * syncCfg2Str(SSyncCfg *pSyncCfg);
+char * syncCfg2SimpleStr(SSyncCfg *pSyncCfg);
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg);
-cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
-char *raftCfg2Str(SRaftCfg *pRaftCfg);
+cJSON * raftCfg2Json(SRaftCfg *pRaftCfg);
+char * raftCfg2Str(SRaftCfg *pRaftCfg);
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg);
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg);
diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h
index b130186516..37f18a6388 100644
--- a/source/libs/sync/inc/syncRaftEntry.h
+++ b/source/libs/sync/inc/syncRaftEntry.h
@@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj);
void syncEntryLog(const SSyncRaftEntry* pObj);
void syncEntryLog2(char* s, const SSyncRaftEntry* pObj);
+//-----------------------------------
+typedef struct SRaftEntryCache {
+ SHashObj* pEntryHash;
+ int32_t maxCount;
+ int32_t currentCount;
+ TdThreadMutex mutex;
+ SSyncNode* pSyncNode;
+} SRaftEntryCache;
+
+SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount);
+void raftCacheDestroy(SRaftEntryCache* pCache);
+int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry);
+int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
+int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
+int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index);
+int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
+int32_t raftCacheClear(struct SRaftEntryCache* pCache);
+
+cJSON* raftCache2Json(SRaftEntryCache* pObj);
+char* raftCache2Str(SRaftEntryCache* pObj);
+void raftCachePrint(SRaftEntryCache* pObj);
+void raftCachePrint2(char* s, SRaftEntryCache* pObj);
+void raftCacheLog(SRaftEntryCache* pObj);
+void raftCacheLog2(char* s, SRaftEntryCache* pObj);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h
index f124867097..f3ed9e302b 100644
--- a/source/libs/sync/inc/syncRaftLog.h
+++ b/source/libs/sync/inc/syncRaftLog.h
@@ -29,9 +29,12 @@ extern "C" {
#include "wal.h"
typedef struct SSyncLogStoreData {
- SSyncNode* pSyncNode;
- SWal* pWal;
+ SSyncNode* pSyncNode;
+ SWal* pWal;
+
+ TdThreadMutex mutex;
SWalReadHandle* pWalHandle;
+
// SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0
} SSyncLogStoreData;
diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c
index c92edae381..982d37826a 100644
--- a/source/libs/sync/src/syncCommit.c
+++ b/source/libs/sync/src/syncCommit.c
@@ -102,6 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
}
}
+ // maybe execute fsm
if (newCommitIndex > pSyncNode->commitIndex) {
SyncIndex beginIndex = pSyncNode->commitIndex + 1;
SyncIndex endIndex = newCommitIndex;
diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c
index 038c36c417..663745a7d7 100644
--- a/source/libs/sync/src/syncIO.c
+++ b/source/libs/sync/src/syncIO.c
@@ -30,7 +30,7 @@ static int32_t syncIODestroy(SSyncIO *io);
static int32_t syncIOStartInternal(SSyncIO *io);
static int32_t syncIOStopInternal(SSyncIO *io);
-static void *syncIOConsumerFunc(void *param);
+static void * syncIOConsumerFunc(void *param);
static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey);
@@ -242,9 +242,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) {
}
static void *syncIOConsumerFunc(void *param) {
- SSyncIO *io = param;
+ SSyncIO * io = param;
STaosQall *qall;
- SRpcMsg *pRpcMsg, rpcMsg;
+ SRpcMsg * pRpcMsg, rpcMsg;
qall = taosAllocateQall();
while (1) {
@@ -281,7 +281,7 @@ static void *syncIOConsumerFunc(void *param) {
if (io->FpOnSyncClientRequest != NULL) {
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
ASSERT(pSyncMsg != NULL);
- io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg);
+ io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg, NULL);
syncClientRequestDestroy(pSyncMsg);
}
diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c
index 5b432aeec4..10f0e0e335 100644
--- a/source/libs/sync/src/syncIndexMgr.c
+++ b/source/libs/sync/src/syncIndexMgr.c
@@ -126,7 +126,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) {
cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index c5af72c971..60f6ca7e7b 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -50,7 +50,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths);
// process message ----
int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
-int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex);
// life cycle
static void syncFreeNode(void* param);
@@ -627,7 +627,7 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) {
return ret;
}
-int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) {
+int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
int32_t ret = 0;
char eventLog[128];
@@ -664,13 +664,34 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak)
SRpcMsg rpcMsg;
syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg);
- if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) {
- ret = 0;
+ // optimized one replica
+ if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) {
+ SyncIndex retIndex;
+ int32_t code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, &retIndex);
+ if (code == 0) {
+ pMsg->info.conn.applyIndex = retIndex;
+ rpcFreeCont(rpcMsg.pCont);
+ syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum);
+ ret = 1;
+ sDebug("vgId:%d optimized index:%ld success, msgtype:%s,%d", pSyncNode->vgId, retIndex,
+ TMSG_INFO(pMsg->msgType), pMsg->msgType);
+ } else {
+ ret = -1;
+ terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
+ sError("vgId:%d optimized index:%ld error, msgtype:%s,%d", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType),
+ pMsg->msgType);
+ }
+
} else {
- ret = -1;
- terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
- sError("syncPropose pSyncNode->FpEqMsg is NULL");
+ if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) {
+ ret = 0;
+ } else {
+ ret = -1;
+ terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
+ sError("enqueue msg error, FpEqMsg is NULL");
+ }
}
+
syncClientRequestDestroy(pSyncMsg);
goto _END;
@@ -914,6 +935,9 @@ void syncNodeStart(SSyncNode* pSyncNode) {
syncNodeBecomeLeader(pSyncNode, "one replica start");
// Raft 3.6.2 Committing entries from previous terms
+ syncNodeAppendNoop(pSyncNode);
+ syncMaybeAdvanceCommitIndex(pSyncNode);
+
return;
}
@@ -1331,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
int32_t userStrLen = strlen(str);
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
- if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
+ if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
}
- SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
- SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
+
+ SyncIndex logLastIndex = SYNC_INDEX_INVALID;
+ SyncIndex logBeginIndex = SYNC_INDEX_INVALID;
+ if (pSyncNode->pLogStore != NULL) {
+ logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+ logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
+ }
char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg));
char* printStr = "";
@@ -1662,6 +1691,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
// change isStandBy to normal (election timeout)
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
syncNodeBecomeLeader(pSyncNode, tmpbuf);
+
+ // Raft 3.6.2 Committing entries from previous terms
+ syncNodeReplicate(pSyncNode);
+ syncNodeAppendNoop(pSyncNode);
+ syncMaybeAdvanceCommitIndex(pSyncNode);
+
} else {
syncNodeBecomeFollower(pSyncNode, tmpbuf);
}
@@ -1807,16 +1842,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) {
// stop elect timer
syncNodeStopElectTimer(pSyncNode);
- // start replicate right now!
- syncNodeReplicate(pSyncNode);
-
// start heartbeat timer
syncNodeStartHeartbeatTimer(pSyncNode);
- // append noop
- syncNodeAppendNoop(pSyncNode);
- syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica
-
// trace log
do {
int32_t debugStrLen = strlen(debugStr);
@@ -1841,9 +1869,9 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode);
// Raft 3.6.2 Committing entries from previous terms
-
- // do not use this
- // syncNodeEqNoop(pSyncNode);
+ syncNodeReplicate(pSyncNode);
+ syncNodeAppendNoop(pSyncNode);
+ syncMaybeAdvanceCommitIndex(pSyncNode);
}
void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
@@ -2375,7 +2403,7 @@ int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
// /\ UNCHANGED <>
//
-int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) {
int32_t ret = 0;
syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg);
@@ -2434,6 +2462,14 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
rpcFreeCont(rpcMsg.pCont);
}
+ if (pRetIndex != NULL) {
+ if (ret == 0 && pEntry != NULL) {
+ *pRetIndex = pEntry->index;
+ } else {
+ *pRetIndex = SYNC_INDEX_INVALID;
+ }
+ }
+
syncEntryDestory(pEntry);
return ret;
}
@@ -2598,6 +2634,10 @@ static int32_t syncNodeProposeConfigChangeFinish(SSyncNode* ths, SyncReconfigFin
return 0;
}
+bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) {
+ return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1);
+}
+
int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) {
int32_t code = 0;
ESyncState state = flag;
@@ -2619,19 +2659,33 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex,
syncEntry2OriginalRpc(pEntry, &rpcMsg);
// user commit
- if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
- SFsmCbMeta cbMeta = {0};
- cbMeta.index = pEntry->index;
- cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
- cbMeta.isWeak = pEntry->isWeak;
- cbMeta.code = 0;
- cbMeta.state = ths->state;
- cbMeta.seqNum = pEntry->seqNum;
- cbMeta.term = pEntry->term;
- cbMeta.currentTerm = ths->pRaftStore->currentTerm;
- cbMeta.flag = flag;
+ if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) {
+ bool internalExecute = true;
+ if ((ths->replicaNum == 1) && ths->restoreFinish && (ths->vgId != 1)) {
+ internalExecute = false;
+ }
- ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta);
+ do {
+ char logBuf[128];
+ snprintf(logBuf, sizeof(logBuf), "index:%ld, internalExecute:%d", i, internalExecute);
+ syncNodeEventLog(ths, logBuf);
+ } while (0);
+
+ // execute fsm in apply thread, or execute outside syncPropose
+ if (internalExecute) {
+ SFsmCbMeta cbMeta = {0};
+ cbMeta.index = pEntry->index;
+ cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index);
+ cbMeta.isWeak = pEntry->isWeak;
+ cbMeta.code = 0;
+ cbMeta.state = ths->state;
+ cbMeta.seqNum = pEntry->seqNum;
+ cbMeta.term = pEntry->term;
+ cbMeta.currentTerm = ths->pRaftStore->currentTerm;
+ cbMeta.flag = flag;
+
+ ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta);
+ }
}
// config change
diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c
index f40eff57d9..9d16bed6c1 100644
--- a/source/libs/sync/src/syncRaftCfg.c
+++ b/source/libs/sync/src/syncRaftCfg.c
@@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char *syncCfg2Str(SSyncCfg *pSyncCfg) {
cJSON *pJson = syncCfg2Json(pSyncCfg);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
@@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
if (pSyncCfg != NULL) {
int32_t len = 512;
- char *s = taosMemoryMalloc(len);
+ char * s = taosMemoryMalloc(len);
memset(s, 0, len);
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
@@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
@@ -271,7 +271,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
}
- cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
+ cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0);
diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c
index ff334a76bb..225360630c 100644
--- a/source/libs/sync/src/syncRaftEntry.c
+++ b/source/libs/sync/src/syncRaftEntry.c
@@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) {
sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized);
taosMemoryFree(serialized);
}
+
+//-----------------------------------
+SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) {
+ SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache));
+ if (pCache == NULL) {
+ sError("vgId:%d raft cache create error", pSyncNode->vgId);
+ return NULL;
+ }
+
+ pCache->pEntryHash =
+ taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ if (pCache->pEntryHash == NULL) {
+ sError("vgId:%d raft cache create hash error", pSyncNode->vgId);
+ return NULL;
+ }
+
+ taosThreadMutexInit(&(pCache->mutex), NULL);
+ pCache->maxCount = maxCount;
+ pCache->currentCount = 0;
+ pCache->pSyncNode = pSyncNode;
+
+ return pCache;
+}
+
+void raftCacheDestroy(SRaftEntryCache* pCache) {
+ if (pCache != NULL) {
+ taosThreadMutexLock(&(pCache->mutex));
+ taosHashCleanup(pCache->pEntryHash);
+ taosThreadMutexUnlock(&(pCache->mutex));
+ taosThreadMutexDestroy(&(pCache->mutex));
+ taosMemoryFree(pCache);
+ }
+}
+
+// success, return 1
+// max count, return 0
+// error, return -1
+int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) {
+ taosThreadMutexLock(&(pCache->mutex));
+
+ if (pCache->currentCount >= pCache->maxCount) {
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 0;
+ }
+
+ taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes);
+ ++(pCache->currentCount);
+
+ do {
+ char eventLog[128];
+ snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d",
+ TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType,
+ pEntry->index, pEntry->bytes);
+ syncNodeEventLog(pCache->pSyncNode, eventLog);
+ } while (0);
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 1;
+}
+
+// success, return 0
+// error, return -1
+// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
+int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
+ if (ppEntry == NULL) {
+ return -1;
+ }
+ *ppEntry = NULL;
+
+ taosThreadMutexLock(&(pCache->mutex));
+ void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
+ if (pTmp != NULL) {
+ SSyncRaftEntry* pEntry = pTmp;
+ *ppEntry = taosMemoryMalloc(pEntry->bytes);
+ memcpy(*ppEntry, pTmp, pEntry->bytes);
+
+ do {
+ char eventLog[128];
+ snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld",
+ TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
+ (*ppEntry)->originalRpcType, (*ppEntry)->index);
+ syncNodeEventLog(pCache->pSyncNode, eventLog);
+ } while (0);
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 0;
+ }
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+ return -1;
+}
+
+// success, return 0
+// error, return -1
+// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
+int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
+ if (ppEntry == NULL) {
+ return -1;
+ }
+ *ppEntry = NULL;
+
+ taosThreadMutexLock(&(pCache->mutex));
+ void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
+ if (pTmp != NULL) {
+ SSyncRaftEntry* pEntry = pTmp;
+ *ppEntry = pEntry;
+
+ do {
+ char eventLog[128];
+ snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld",
+ TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
+ (*ppEntry)->originalRpcType, (*ppEntry)->index);
+ syncNodeEventLog(pCache->pSyncNode, eventLog);
+ } while (0);
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 0;
+ }
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+ return -1;
+}
+
+int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) {
+ taosThreadMutexLock(&(pCache->mutex));
+ taosHashRemove(pCache->pEntryHash, &index, sizeof(index));
+ --(pCache->currentCount);
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 0;
+}
+
+int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
+ if (ppEntry == NULL) {
+ return -1;
+ }
+ *ppEntry = NULL;
+
+ taosThreadMutexLock(&(pCache->mutex));
+ void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
+ if (pTmp != NULL) {
+ SSyncRaftEntry* pEntry = pTmp;
+ *ppEntry = taosMemoryMalloc(pEntry->bytes);
+ memcpy(*ppEntry, pTmp, pEntry->bytes);
+
+ do {
+ char eventLog[128];
+ snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld",
+ TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
+ (*ppEntry)->originalRpcType, (*ppEntry)->index);
+ syncNodeEventLog(pCache->pSyncNode, eventLog);
+ } while (0);
+
+ taosHashRemove(pCache->pEntryHash, &index, sizeof(index));
+ --(pCache->currentCount);
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 0;
+ }
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+ return -1;
+}
+
+int32_t raftCacheClear(struct SRaftEntryCache* pCache) {
+ taosThreadMutexLock(&(pCache->mutex));
+ taosHashClear(pCache->pEntryHash);
+ pCache->currentCount = 0;
+ taosThreadMutexUnlock(&(pCache->mutex));
+ return 0;
+}
+
+//-----------------------------------
+cJSON* raftCache2Json(SRaftEntryCache* pCache) {
+ char u64buf[128] = {0};
+ cJSON* pRoot = cJSON_CreateObject();
+
+ if (pCache != NULL) {
+ taosThreadMutexLock(&(pCache->mutex));
+
+ snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode);
+ cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+ cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount);
+ cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount);
+ cJSON* pEntries = cJSON_CreateArray();
+ cJSON_AddItemToObject(pRoot, "entries", pEntries);
+
+ SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL);
+ if (pIter != NULL) {
+ SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter;
+ cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
+ }
+ while (pIter) {
+ pIter = taosHashIterate(pCache->pEntryHash, pIter);
+ if (pIter != NULL) {
+ SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter;
+ cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
+ }
+ }
+
+ taosThreadMutexUnlock(&(pCache->mutex));
+ }
+
+ cJSON* pJson = cJSON_CreateObject();
+ cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot);
+ return pJson;
+}
+
+char* raftCache2Str(SRaftEntryCache* pCache) {
+ cJSON* pJson = raftCache2Json(pCache);
+ char* serialized = cJSON_Print(pJson);
+ cJSON_Delete(pJson);
+ return serialized;
+}
+
+void raftCachePrint(SRaftEntryCache* pCache) {
+ char* serialized = raftCache2Str(pCache);
+ printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void raftCachePrint2(char* s, SRaftEntryCache* pCache) {
+ char* serialized = raftCache2Str(pCache);
+ printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void raftCacheLog(SRaftEntryCache* pCache) {
+ char* serialized = raftCache2Str(pCache);
+ sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized);
+ taosMemoryFree(serialized);
+}
+
+void raftCacheLog2(char* s, SRaftEntryCache* pCache) {
+ if (gRaftDetailLog) {
+ char* serialized = raftCache2Str(pCache);
+ sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+ taosMemoryFree(serialized);
+ }
+}
\ No newline at end of file
diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c
index cc606f4df8..d84c576714 100644
--- a/source/libs/sync/src/syncRaftLog.c
+++ b/source/libs/sync/src/syncRaftLog.c
@@ -257,6 +257,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
return -1;
}
+ taosThreadMutexLock(&(pData->mutex));
+
code = walReadWithHandle(pWalHandle, index);
if (code != 0) {
int32_t err = terrno;
@@ -281,6 +283,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
terrno = saveErr;
*/
+ taosThreadMutexUnlock(&(pData->mutex));
return code;
}
@@ -301,6 +304,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
terrno = saveErr;
*/
+ taosThreadMutexUnlock(&(pData->mutex));
return code;
}
@@ -364,6 +368,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
pData->pWal = pSyncNode->pWal;
ASSERT(pData->pWal != NULL);
+ taosThreadMutexInit(&(pData->mutex), NULL);
pData->pWalHandle = walOpenReadHandle(pData->pWal);
ASSERT(pData->pWalHandle != NULL);
@@ -408,9 +413,14 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
void logStoreDestory(SSyncLogStore* pLogStore) {
if (pLogStore != NULL) {
SSyncLogStoreData* pData = pLogStore->data;
+
+ taosThreadMutexLock(&(pData->mutex));
if (pData->pWalHandle != NULL) {
walCloseReadHandle(pData->pWalHandle);
+ pData->pWalHandle = NULL;
}
+ taosThreadMutexUnlock(&(pData->mutex));
+ taosThreadMutexDestroy(&(pData->mutex));
taosMemoryFree(pLogStore->data);
taosMemoryFree(pLogStore);
@@ -460,6 +470,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
SWal* pWal = pData->pWal;
if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
+ taosThreadMutexLock(&(pData->mutex));
+
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
ASSERT(pWalHandle != NULL);
@@ -503,6 +515,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
terrno = saveErr;
*/
+ taosThreadMutexUnlock(&(pData->mutex));
return pEntry;
} else {
diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c
index a1ab95c00f..9f5cba6c66 100644
--- a/source/libs/sync/src/syncRaftStore.c
+++ b/source/libs/sync/src/syncRaftStore.c
@@ -216,7 +216,7 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) {
char *raftStore2Str(SRaftStore *pRaftStore) {
cJSON *pJson = raftStore2Json(pRaftStore);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c
index 48c1b70a04..d2cbabe226 100644
--- a/source/libs/sync/src/syncRespMgr.c
+++ b/source/libs/sync/src/syncRespMgr.c
@@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
}
void syncRespMgrDestroy(SSyncRespMgr *pObj) {
- taosThreadMutexLock(&(pObj->mutex));
- taosHashCleanup(pObj->pRespHash);
- taosThreadMutexUnlock(&(pObj->mutex));
- taosThreadMutexDestroy(&(pObj->mutex));
- taosMemoryFree(pObj);
+ if (pObj != NULL) {
+ taosThreadMutexLock(&(pObj->mutex));
+ taosHashCleanup(pObj->pRespHash);
+ taosThreadMutexUnlock(&(pObj->mutex));
+ taosThreadMutexDestroy(&(pObj->mutex));
+ taosMemoryFree(pObj);
+ }
}
int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) {
diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c
index 4fc0128400..0feaf532bb 100644
--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@@ -314,14 +314,14 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
char *snapshotSender2Str(SSyncSnapshotSender *pSender) {
cJSON *pJson = snapshotSender2Json(pSender);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) {
int32_t len = 256;
- char *s = taosMemoryMalloc(len);
+ char * s = taosMemoryMalloc(len);
SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
char host[128];
@@ -461,7 +461,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
cJSON_AddStringToObject(pFromId, "addr", u64buf);
{
uint64_t u64 = pReceiver->fromId.addr;
- cJSON *pTmp = pFromId;
+ cJSON * pTmp = pFromId;
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
@@ -494,14 +494,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) {
cJSON *pJson = snapshotReceiver2Json(pReceiver);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) {
int32_t len = 256;
- char *s = taosMemoryMalloc(len);
+ char * s = taosMemoryMalloc(len);
SRaftId fromId = pReceiver->fromId;
char host[128];
diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c
index d6c2cbd34e..2c43312064 100644
--- a/source/libs/sync/src/syncVoteMgr.c
+++ b/source/libs/sync/src/syncVoteMgr.c
@@ -127,7 +127,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
char *voteGranted2Str(SVotesGranted *pVotesGranted) {
cJSON *pJson = voteGranted2Json(pVotesGranted);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
@@ -256,7 +256,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) {
char *votesRespond2Str(SVotesRespond *pVotesRespond) {
cJSON *pJson = votesRespond2Json(pVotesRespond);
- char *serialized = cJSON_Print(pJson);
+ char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 27084286da..bc63462a9e 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "")
add_executable(syncIndexMgrTest "")
add_executable(syncLogStoreTest "")
add_executable(syncEntryTest "")
+add_executable(syncEntryCacheTest "")
add_executable(syncRequestVoteTest "")
add_executable(syncRequestVoteReplyTest "")
add_executable(syncAppendEntriesTest "")
@@ -129,6 +130,10 @@ target_sources(syncEntryTest
PRIVATE
"syncEntryTest.cpp"
)
+target_sources(syncEntryCacheTest
+ PRIVATE
+ "syncEntryCacheTest.cpp"
+)
target_sources(syncRequestVoteTest
PRIVATE
"syncRequestVoteTest.cpp"
@@ -362,6 +367,11 @@ target_include_directories(syncEntryTest
"${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
+target_include_directories(syncEntryCacheTest
+ PUBLIC
+ "${TD_SOURCE_DIR}/include/libs/sync"
+ "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
target_include_directories(syncRequestVoteTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/sync"
@@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest
sync
gtest_main
)
+target_link_libraries(syncEntryCacheTest
+ sync
+ gtest_main
+)
target_link_libraries(syncRequestVoteTest
sync
gtest_main
diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp
new file mode 100644
index 0000000000..787c08e507
--- /dev/null
+++ b/source/libs/sync/test/syncEntryCacheTest.cpp
@@ -0,0 +1,162 @@
+#include
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftLog.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+
+void logTest() {
+ sTrace("--- sync log test: trace");
+ sDebug("--- sync log test: debug");
+ sInfo("--- sync log test: info");
+ sWarn("--- sync log test: warn");
+ sError("--- sync log test: error");
+ sFatal("--- sync log test: fatal");
+}
+
+SSyncRaftEntry* createEntry(int i) {
+ int32_t dataLen = 20;
+ SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
+ assert(pEntry != NULL);
+ pEntry->msgType = 88;
+ pEntry->originalRpcType = 99;
+ pEntry->seqNum = 3;
+ pEntry->isWeak = true;
+ pEntry->term = 100 + i;
+ pEntry->index = i;
+ snprintf(pEntry->data, dataLen, "value%d", i);
+
+ return pEntry;
+}
+
+SSyncNode* createFakeNode() {
+ SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode));
+ ASSERT(pSyncNode != NULL);
+ memset(pSyncNode, 0, sizeof(SSyncNode));
+
+ return pSyncNode;
+}
+
+SRaftEntryCache* createCache(int maxCount) {
+ SSyncNode* pSyncNode = createFakeNode();
+ ASSERT(pSyncNode != NULL);
+
+ SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount);
+ ASSERT(pCache != NULL);
+
+ return pCache;
+}
+
+void test1() {
+ int32_t code = 0;
+ SRaftEntryCache* pCache = createCache(5);
+ for (int i = 0; i < 5; ++i) {
+ SSyncRaftEntry* pEntry = createEntry(i);
+ code = raftCachePutEntry(pCache, pEntry);
+ ASSERT(code == 1);
+ syncEntryDestory(pEntry);
+ }
+ raftCacheLog2((char*)"==test1 write 5 entries==", pCache);
+
+ SyncIndex index;
+ index = 1;
+ code = raftCacheDelEntry(pCache, index);
+ ASSERT(code == 0);
+ index = 3;
+ code = raftCacheDelEntry(pCache, index);
+ ASSERT(code == 0);
+ raftCacheLog2((char*)"==test1 delete 1,3==", pCache);
+
+ code = raftCacheClear(pCache);
+ ASSERT(code == 0);
+ raftCacheLog2((char*)"==clear all==", pCache);
+}
+
+void test2() {
+ int32_t code = 0;
+ SRaftEntryCache* pCache = createCache(5);
+ for (int i = 0; i < 5; ++i) {
+ SSyncRaftEntry* pEntry = createEntry(i);
+ code = raftCachePutEntry(pCache, pEntry);
+ ASSERT(code == 1);
+ syncEntryDestory(pEntry);
+ }
+ raftCacheLog2((char*)"==test2 write 5 entries==", pCache);
+
+ SyncIndex index;
+ index = 1;
+ SSyncRaftEntry* pEntry;
+ code = raftCacheGetEntry(pCache, index, &pEntry);
+ ASSERT(code == 0);
+ syncEntryDestory(pEntry);
+ syncEntryLog2((char*)"==test2 get entry 1==", pEntry);
+
+ index = 2;
+ code = raftCacheGetEntryP(pCache, index, &pEntry);
+ ASSERT(code == 0);
+ syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry);
+
+ // not found
+ index = 8;
+ code = raftCacheGetEntry(pCache, index, &pEntry);
+ ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
+ sTrace("==test2 get entry 8 not found==");
+
+ // not found
+ index = 9;
+ code = raftCacheGetEntryP(pCache, index, &pEntry);
+ ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
+ sTrace("==test2 get entry pointer 9 not found==");
+}
+
+void test3() {
+ int32_t code = 0;
+ SRaftEntryCache* pCache = createCache(5);
+ for (int i = 0; i < 5; ++i) {
+ SSyncRaftEntry* pEntry = createEntry(i);
+ code = raftCachePutEntry(pCache, pEntry);
+ ASSERT(code == 1);
+ syncEntryDestory(pEntry);
+ }
+ for (int i = 6; i < 10; ++i) {
+ SSyncRaftEntry* pEntry = createEntry(i);
+ code = raftCachePutEntry(pCache, pEntry);
+ ASSERT(code == 0);
+ syncEntryDestory(pEntry);
+ }
+ raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache);
+}
+
+void test4() {
+ int32_t code = 0;
+ SRaftEntryCache* pCache = createCache(5);
+ for (int i = 0; i < 5; ++i) {
+ SSyncRaftEntry* pEntry = createEntry(i);
+ code = raftCachePutEntry(pCache, pEntry);
+ ASSERT(code == 1);
+ syncEntryDestory(pEntry);
+ }
+ raftCacheLog2((char*)"==test4 write 5 entries==", pCache);
+
+ SyncIndex index;
+ index = 3;
+ SSyncRaftEntry* pEntry;
+ code = raftCacheGetAndDel(pCache, index, &pEntry);
+ ASSERT(code == 0);
+ syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry);
+ raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache);
+}
+
+int main(int argc, char** argv) {
+ gRaftDetailLog = true;
+ tsAsyncLog = 0;
+ sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG;
+
+ test1();
+ test2();
+ test3();
+ test4();
+
+ return 0;
+}
diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp
index e3ceae0615..9e0c2ecc29 100644
--- a/source/libs/sync/test/syncRaftLogTest2.cpp
+++ b/source/libs/sync/test/syncRaftLogTest2.cpp
@@ -113,7 +113,7 @@ void test2() {
pLogStore = logStoreCreate(pSyncNode);
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
- //pLogStore->syncLogSetBeginIndex(pLogStore, 5);
+ // pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore);
@@ -229,7 +229,7 @@ void test4() {
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore);
- //pLogStore->syncLogSetBeginIndex(pLogStore, 5);
+ // pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) {
@@ -291,7 +291,7 @@ void test5() {
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore);
- //pLogStore->syncLogSetBeginIndex(pLogStore, 5);
+ // pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) {
@@ -412,26 +412,23 @@ void test6() {
do {
SyncIndex firstVer = walGetFirstVer(pWal);
SyncIndex lastVer = walGetLastVer(pWal);
- bool isEmpty = walIsEmpty(pWal);
+ bool isEmpty = walIsEmpty(pWal);
printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty);
} while (0);
logStoreDestory(pLogStore);
cleanup();
-
-
// restart
init();
pLogStore = logStoreCreate(pSyncNode);
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
-
do {
SyncIndex firstVer = walGetFirstVer(pWal);
SyncIndex lastVer = walGetLastVer(pWal);
- bool isEmpty = walIsEmpty(pWal);
+ bool isEmpty = walIsEmpty(pWal);
printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty);
} while (0);
@@ -461,13 +458,13 @@ int main(int argc, char** argv) {
}
sTrace("gAssert : %d", gAssert);
-/*
- test1();
- test2();
- test3();
- test4();
- test5();
-*/
+ /*
+ test1();
+ test2();
+ test3();
+ test4();
+ test5();
+ */
test6();
return 0;
diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp
index 302e29a091..ea1788c545 100644
--- a/source/libs/sync/test/syncRaftLogTest3.cpp
+++ b/source/libs/sync/test/syncRaftLogTest3.cpp
@@ -312,7 +312,7 @@ void test5() {
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore);
- //pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6);
+ // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6);
pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5);
for (int i = 6; i <= 10; ++i) {
int32_t dataLen = 10;
diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h
index bbd9ce3a78..963a85922f 100644
--- a/source/libs/transport/inc/transComm.h
+++ b/source/libs/transport/inc/transComm.h
@@ -105,6 +105,13 @@ typedef SRpcCtxVal STransCtxVal;
typedef SRpcInfo STrans;
typedef SRpcConnInfo STransHandleInfo;
+// ref mgt
+// handle
+typedef struct SExHandle {
+ void* handle;
+ int64_t refId;
+ void* pThrd;
+} SExHandle;
/*convet from fqdn to ip */
typedef struct SCvtAddr {
char ip[TSDB_FQDN_LEN];
@@ -113,14 +120,15 @@ typedef struct SCvtAddr {
} SCvtAddr;
typedef struct {
- SEpSet epSet; // ip list provided by app
- SEpSet origEpSet;
- void* ahandle; // handle provided by app
- tmsg_t msgType; // message type
- int8_t connType; // connection type cli/srv
- int64_t rid; // refId returned by taosAddRef
+ SEpSet epSet; // ip list provided by app
+ SEpSet origEpSet;
+ void* ahandle; // handle provided by app
+ tmsg_t msgType; // message type
+ int8_t connType; // connection type cli/srv
- int8_t retryCount;
+ int8_t retryCnt;
+ int8_t retryLimit;
+ // bool setMaxRetry;
STransCtx appCtx; //
STransMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
@@ -239,6 +247,32 @@ int transSendAsync(SAsyncPool* pool, queue* mq);
} \
} \
} while (0)
+
+#define ASYNC_CHECK_HANDLE(exh1, id) \
+ do { \
+ if (id > 0) { \
+ tTrace("handle step1"); \
+ SExHandle* exh2 = transAcquireExHandle(refMgt, id); \
+ if (exh2 == NULL || id != exh2->refId) { \
+ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
+ exh2 ? exh2->refId : 0, id); \
+ goto _return1; \
+ } \
+ } else if (id == 0) { \
+ tTrace("handle step2"); \
+ SExHandle* exh2 = transAcquireExHandle(refMgt, id); \
+ if (exh2 == NULL || id == exh2->refId) { \
+ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \
+ exh2 ? exh2->refId : 0); \
+ goto _return1; \
+ } else { \
+ id = exh1->refId; \
+ } \
+ } else if (id < 0) { \
+ tTrace("handle step3"); \
+ goto _return2; \
+ } \
+ } while (0)
int transInitBuffer(SConnBuffer* buf);
int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf);
@@ -349,21 +383,13 @@ void transDQDestroy(SDelayQueue* queue);
int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
-void transPrintEpSet(SEpSet* pEpSet);
+// void transPrintEpSet(SEpSet* pEpSet);
bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
/*
* init global func
*/
void transThreadOnce();
-// ref mgt
-// handle
-typedef struct SExHandle {
- void* handle;
- int64_t refId;
- void* pThrd;
-} SExHandle;
-
void transInitEnv();
int32_t transOpenExHandleMgt(int size);
void transCloseExHandleMgt(int32_t mgt);
diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c
index 1ec96f4a7a..4f7b19b539 100644
--- a/source/libs/transport/src/trans.c
+++ b/source/libs/transport/src/trans.c
@@ -79,6 +79,7 @@ void* rpcOpen(const SRpcInit* pInit) {
return pRpc;
}
void rpcClose(void* arg) {
+ tInfo("start to close rpc");
SRpcInfo* pRpc = (SRpcInfo*)arg;
(*taosCloseHandle[pRpc->connType])(pRpc->tcphandle);
transCloseExHandleMgt(pRpc->refMgt);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index e563bf0ddd..7374d1fffc 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2019 TAOS Data, Inc.
+/** Copyright (c) 2019 TAOS Data, Inc.
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
@@ -25,7 +25,6 @@ typedef struct SCliConn {
uv_write_t writeReq;
void* hostThrd;
- int hThrdIdx;
SConnBuffer readBuf;
STransQueue cliMsgs;
@@ -36,6 +35,7 @@ typedef struct SCliConn {
bool broken; // link broken or not
ConnStatus status; //
+ int64_t refId;
char* ip;
uint32_t port;
@@ -54,7 +54,7 @@ typedef struct SCliMsg {
int sent; //(0: no send, 1: alread sent)
} SCliMsg;
-typedef struct SCliThrdObj {
+typedef struct SCliThrd {
TdThread thread; // tid
int64_t pid; // pid
uv_loop_t* loop;
@@ -72,13 +72,13 @@ typedef struct SCliThrdObj {
SCvtAddr cvtAddr;
bool quit;
-} SCliThrdObj;
+} SCliThrd;
typedef struct SCliObj {
- char label[TSDB_LABEL_LEN];
- int32_t index;
- int numOfThreads;
- SCliThrdObj** pThreadObj;
+ char label[TSDB_LABEL_LEN];
+ int32_t index;
+ int numOfThreads;
+ SCliThrd** pThreadObj;
} SCliObj;
typedef struct SConnList {
@@ -106,11 +106,18 @@ static void cliAsyncCb(uv_async_t* handle);
static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg);
-static SCliConn* cliCreateConn(SCliThrdObj* thrd);
+static SCliConn* cliCreateConn(SCliThrd* thrd);
static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
static void cliDestroy(uv_handle_t* handle);
static void cliSend(SCliConn* pConn);
+static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
+ if (code != 0) return false;
+ if (pCtx->retryCnt == 0) return false;
+ if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false;
+ return true;
+}
+
void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
/*
* set TCP connection timeout per-socket level
@@ -122,14 +129,14 @@ static void cliHandleResp(SCliConn* conn);
static void cliHandleExcept(SCliConn* conn);
// handle req from app
-static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd);
-static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease,
- NULL, cliHandleUpdate};
+static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd);
+static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd);
+static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd);
+static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd);
+static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL,
+ cliHandleUpdate};
-static void cliSendQuit(SCliThrdObj* thrd);
+static void cliSendQuit(SCliThrd* thrd);
static void destroyUserdata(STransMsg* userdata);
static int cliRBChoseIdx(STrans* pTransInst);
@@ -137,8 +144,8 @@ static int cliRBChoseIdx(STrans* pTransInst);
static void destroyCmsg(SCliMsg* cmsg);
static void transDestroyConnCtx(STransConnCtx* ctx);
// thread obj
-static SCliThrdObj* createThrdObj();
-static void destroyThrdObj(SCliThrdObj* pThrd);
+static SCliThrd* createThrdObj();
+static void destroyThrdObj(SCliThrd* pThrd);
static void cliWalkCb(uv_handle_t* handle, void* arg);
@@ -154,7 +161,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
destroyCmsg(pMsg);
}
}
-
#define CLI_RELEASE_UV(loop) \
do { \
uv_walk(loop, cliWalkCb, NULL); \
@@ -168,17 +174,23 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \
} while (0)
-#define CONN_HOST_THREAD_IDX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1)
+#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \
+ do { \
+ if (exh == NULL) { \
+ idx = -1; \
+ } else { \
+ ASYNC_CHECK_HANDLE((exh), refId); \
+ pThrd = (SCliThrd*)(exh)->pThrd; \
+ } \
+ } while (0)
#define CONN_PERSIST_TIME(para) (para * 1000 * 10)
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
-#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
+#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label)
#define CONN_SHOULD_RELEASE(conn, head) \
do { \
if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \
- int connStatus = conn->status; \
uint64_t ahandle = head->ahandle; \
CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \
- conn->status = ConnRelease; \
transClearBuffer(&conn->readBuf); \
transFreeMsg(transContFromHead((char*)head)); \
tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \
@@ -187,9 +199,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
} \
destroyCmsg(pMsg); \
cliReleaseUnfinishedMsg(conn); \
- if (connStatus != ConnInPool) { \
- addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \
- } \
+ addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \
return; \
} \
} while (0)
@@ -255,8 +265,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
#define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1)
#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release)
+#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps
#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn)
#define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port)
+#define EPSET_FORWARD_INUSE(epSet) \
+ do { \
+ (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \
+ } while (0)
+#define EPSET_DEBUG_STR(epSet, tbuf) \
+ do { \
+ int len = snprintf(tbuf, sizeof(tbuf), "epset:{"); \
+ for (int i = 0; i < (epSet)->numOfEps; i++) { \
+ if (i == (epSet)->numOfEps - 1) { \
+ len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \
+ } else { \
+ len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \
+ } \
+ } \
+ len += snprintf(tbuf + len, sizeof(tbuf) - len, "}, inUse:%d", (epSet)->inUse); \
+ } while (0);
static void* cliWorkThread(void* arg);
@@ -271,8 +298,8 @@ _RETURN:
return false;
}
void cliHandleResp(SCliConn* conn) {
- SCliThrdObj* pThrd = conn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
+ SCliThrd* pThrd = conn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
pHead->code = htonl(pHead->code);
@@ -292,17 +319,9 @@ void cliHandleResp(SCliConn* conn) {
if (CONN_NO_PERSIST_BY_APP(conn)) {
pMsg = transQueuePop(&conn->cliMsgs);
- pCtx = pMsg ? pMsg->ctx : NULL;
- if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) {
- transMsg.info.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType);
- if (transMsg.info.ahandle == NULL) {
- transMsg.info.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType));
- }
- tDebug("%s conn %p construct ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
- } else {
- transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
- tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
- }
+ pCtx = pMsg->ctx;
+ transMsg.info.ahandle = pCtx->ahandle;
+ tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
} else {
uint64_t ahandle = (uint64_t)pHead->ahandle;
CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle);
@@ -324,26 +343,22 @@ void cliHandleResp(SCliConn* conn) {
}
// buf's mem alread translated to transMsg.pCont
transClearBuffer(&conn->readBuf);
-
if (!CONN_NO_PERSIST_BY_APP(conn)) {
- transMsg.info.handle = conn;
+ transMsg.info.handle = (void*)conn->refId;
tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn);
}
- // char buf[64] = {0};
- // TRACE_TO_STR(&transMsg.info.traceId, buf);
+
STraceId* trace = &transMsg.info.traceId;
- tGTrace("conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", conn, TMSG_INFO(pHead->msgType),
- taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr),
- ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code);
+ tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn),
+ conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
+ taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code);
if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) {
- tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn));
- // transUnrefCliHandle(conn);
+ tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
return;
}
if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) {
- tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn));
- // transUnrefCliHandle(conn);
+ tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
return;
}
@@ -375,9 +390,9 @@ void cliHandleExcept(SCliConn* pConn) {
return;
}
}
- SCliThrdObj* pThrd = pConn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
- bool once = false;
+ SCliThrd* pThrd = pConn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
+ bool once = false;
do {
SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs);
if (pMsg == NULL && once) {
@@ -389,7 +404,6 @@ void cliHandleExcept(SCliConn* pConn) {
transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0;
transMsg.info.ahandle = NULL;
- transMsg.info.handle = pConn;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType);
@@ -414,15 +428,15 @@ void cliHandleExcept(SCliConn* pConn) {
return;
}
destroyCmsg(pMsg);
- tTrace("%s conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn);
+ tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn));
} while (!transQueueEmpty(&pConn->cliMsgs));
transUnrefCliHandle(pConn);
}
void cliTimeoutCb(uv_timer_t* handle) {
- SCliThrdObj* pThrd = handle->data;
- STrans* pTransInst = pThrd->pTransInst;
- int64_t currentTime = pThrd->nextTimeout;
+ SCliThrd* pThrd = handle->data;
+ STrans* pTransInst = pThrd->pTransInst;
+ int64_t currentTime = pThrd->nextTimeout;
tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label);
SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
@@ -487,10 +501,26 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
assert(h == &conn->conn);
return conn;
}
+static void allocConnRef(SCliConn* conn, bool update) {
+ if (update) {
+ transRemoveExHandle(refMgt, conn->refId);
+ }
+ SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
+ exh->handle = conn;
+ exh->pThrd = conn->hostThrd;
+ exh->refId = transAddExHandle(refMgt, exh);
+ conn->refId = exh->refId;
+}
static void addConnToPool(void* pool, SCliConn* conn) {
- SCliThrdObj* thrd = conn->hostThrd;
+ if (conn->status == ConnInPool) {
+ // assert(0);
+ return;
+ }
+ SCliThrd* thrd = conn->hostThrd;
CONN_HANDLE_THREAD_QUIT(thrd);
+ allocConnRef(conn, true);
+
STrans* pTransInst = thrd->pTransInst;
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
transQueueClear(&conn->cliMsgs);
@@ -499,7 +529,7 @@ static void addConnToPool(void* pool, SCliConn* conn) {
char key[128] = {0};
CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port);
- tTrace("%s conn %p added to conn pool, read buf cap: %d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap);
+ tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
// list already create before
@@ -540,13 +570,14 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
return;
}
if (nread < 0) {
- tError("%s conn %p read error: %s", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread));
+ tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
+ T_REF_VAL_GET(conn));
conn->broken = true;
cliHandleExcept(conn);
}
}
-static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
+static SCliConn* cliCreateConn(SCliThrd* pThrd) {
SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn));
// read/write stream handle
conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t));
@@ -562,11 +593,16 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
conn->status = ConnNormal;
conn->broken = 0;
transRefCliHandle(conn);
+
+ allocConnRef(conn, false);
+
return conn;
}
static void cliDestroyConn(SCliConn* conn, bool clear) {
tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn);
QUEUE_REMOVE(&conn->conn);
+ QUEUE_INIT(&conn->conn);
+ transRemoveExHandle(refMgt, conn->refId);
if (clear) {
uv_close((uv_handle_t*)conn->stream, cliDestroy);
}
@@ -593,7 +629,7 @@ static bool cliHandleNoResp(SCliConn* conn) {
}
if (res == true) {
if (cliMaySendCachedMsg(conn) == false) {
- SCliThrdObj* thrd = conn->hostThrd;
+ SCliThrd* thrd = conn->hostThrd;
addConnToPool(thrd->pool, conn);
}
}
@@ -629,8 +665,8 @@ void cliSend(SCliConn* pConn) {
STransConnCtx* pCtx = pCliMsg->ctx;
- SCliThrdObj* pThrd = pConn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
+ SCliThrd* pThrd = pConn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg);
if (pMsg->pCont == 0) {
@@ -651,12 +687,10 @@ void cliSend(SCliConn* pConn) {
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
- // char buf[64] = {0};
- // TRACE_TO_STR(&pMsg->info.traceId, buf);
STraceId* trace = &pMsg->info.traceId;
- tGTrace("conn %p %s is sent to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType),
- taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr),
- ntohs(pConn->localAddr.sin_port));
+ tGTrace("%s conn %p %s is sent to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn,
+ TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
+ taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port));
if (pHead->persist == 1) {
CONN_SET_PERSIST_BY_APP(pConn);
@@ -664,7 +698,6 @@ void cliSend(SCliConn* pConn) {
pConn->writeReq.data = pConn;
uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
-
return;
_RETURN:
return;
@@ -690,19 +723,24 @@ void cliConnCb(uv_connect_t* req, int status) {
cliSend(pConn);
}
-static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) {
+ pThrd->quit = true;
tDebug("cli work thread %p start to quit", pThrd);
destroyCmsg(pMsg);
destroyConnPool(pThrd->pool);
uv_timer_stop(&pThrd->timer);
uv_walk(pThrd->loop, cliWalkCb, NULL);
- pThrd->quit = true;
-
// uv_stop(pThrd->loop);
}
-static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) {
- SCliConn* conn = pMsg->msg.info.handle;
+static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) {
+ int64_t refId = (int64_t)(pMsg->msg.info.handle);
+ SExHandle* exh = transAcquireExHandle(refMgt, refId);
+ if (exh == NULL) {
+ tDebug("%" PRId64 " already release", refId);
+ }
+
+ SCliConn* conn = exh->handle;
tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn);
if (T_REF_VAL_GET(conn) == 2) {
@@ -711,33 +749,37 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) {
return;
}
cliSend(conn);
- } else {
- // conn already broken down
- transUnrefCliHandle(conn);
}
}
-static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) {
STransConnCtx* pCtx = pMsg->ctx;
-
pThrd->cvtAddr = pCtx->cvtAddr;
destroyCmsg(pMsg);
}
-SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
SCliConn* conn = NULL;
- if (pMsg->msg.info.handle != NULL) {
- conn = (SCliConn*)(pMsg->msg.info.handle);
- if (conn != NULL) {
- tTrace("%s conn %p reused", CONN_GET_INST_LABEL(conn), conn);
- }
- } else {
- STransConnCtx* pCtx = pMsg->ctx;
- conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet));
- if (conn != NULL) {
- tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn);
+ int64_t refId = (int64_t)(pMsg->msg.info.handle);
+ if (refId != 0) {
+ SExHandle* exh = transAcquireExHandle(refMgt, refId);
+ if (exh == NULL) {
+ *ignore = true;
+ destroyCmsg(pMsg);
+ return NULL;
+ // assert(0);
} else {
- tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool);
+ conn = exh->handle;
+ transReleaseExHandle(refMgt, refId);
}
+ return conn;
+ };
+
+ STransConnCtx* pCtx = pMsg->ctx;
+ conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet));
+ if (conn != NULL) {
+ tTrace("%s conn %p get from conn pool:%p", CONN_GET_INST_LABEL(conn), conn, pThrd->pool);
+ } else {
+ tTrace("%s not found conn in conn pool:%p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool);
}
return conn;
}
@@ -752,22 +794,19 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
}
}
}
-void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
- uint64_t et = taosGetTimestampUs();
- uint64_t el = et - pMsg->st;
- // tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el);
-
+void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
STransConnCtx* pCtx = pMsg->ctx;
STrans* pTransInst = pThrd->pTransInst;
cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr);
- transPrintEpSet(&pCtx->epSet);
-
- SCliConn* conn = cliGetConn(pMsg, pThrd);
+ // transPrintEpSet(&pCtx->epSet);
+ bool ignore = false;
+ SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore);
+ if (ignore == true) {
+ return;
+ }
if (conn != NULL) {
- conn->hThrdIdx = pCtx->hThrdIdx;
-
transCtxMerge(&conn->ctx, &pCtx->appCtx);
transQueuePush(&conn->cliMsgs, pMsg);
cliSend(conn);
@@ -776,7 +815,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
transCtxMerge(&conn->ctx, &pCtx->appCtx);
transQueuePush(&conn->cliMsgs, pMsg);
- conn->hThrdIdx = pCtx->hThrdIdx;
conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet));
conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet);
@@ -784,7 +822,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
if (ret) {
tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret));
}
- int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT);
+ int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT);
if (fd == -1) {
tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn);
cliHandleExcept(conn);
@@ -807,9 +845,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
}
}
static void cliAsyncCb(uv_async_t* handle) {
- SAsyncItem* item = handle->data;
- SCliThrdObj* pThrd = item->pThrd;
- SCliMsg* pMsg = NULL;
+ SAsyncItem* item = handle->data;
+ SCliThrd* pThrd = item->pThrd;
+ SCliMsg* pMsg = NULL;
// batch process to avoid to lock/unlock frequently
queue wq;
@@ -835,7 +873,7 @@ static void cliAsyncCb(uv_async_t* handle) {
}
static void* cliWorkThread(void* arg) {
- SCliThrdObj* pThrd = (SCliThrdObj*)arg;
+ SCliThrd* pThrd = (SCliThrd*)arg;
pThrd->pid = taosGetSelfPthreadId();
setThreadName("trans-cli-work");
uv_run(pThrd->loop, UV_RUN_DEFAULT);
@@ -848,10 +886,10 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
STrans* pTransInst = shandle;
memcpy(cli->label, label, strlen(label));
cli->numOfThreads = numOfThreads;
- cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*));
+ cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*));
for (int i = 0; i < cli->numOfThreads; i++) {
- SCliThrdObj* pThrd = createThrdObj();
+ SCliThrd* pThrd = createThrdObj();
pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
pThrd->pTransInst = shandle;
@@ -885,8 +923,8 @@ static void destroyCmsg(SCliMsg* pMsg) {
taosMemoryFree(pMsg);
}
-static SCliThrdObj* createThrdObj() {
- SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj));
+static SCliThrd* createThrdObj() {
+ SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd));
QUEUE_INIT(&pThrd->msg);
taosThreadMutexInit(&pThrd->msgMtx, NULL);
@@ -904,7 +942,7 @@ static SCliThrdObj* createThrdObj() {
pThrd->quit = false;
return pThrd;
}
-static void destroyThrdObj(SCliThrdObj* pThrd) {
+static void destroyThrdObj(SCliThrd* pThrd) {
if (pThrd == NULL) {
return;
}
@@ -925,7 +963,7 @@ static void transDestroyConnCtx(STransConnCtx* ctx) {
taosMemoryFree(ctx);
}
-void cliSendQuit(SCliThrdObj* thrd) {
+void cliSendQuit(SCliThrd* thrd) {
// cli can stop gracefully
SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg));
msg->type = Quit;
@@ -938,7 +976,10 @@ void cliWalkCb(uv_handle_t* handle, void* arg) {
}
int cliRBChoseIdx(STrans* pTransInst) {
- int64_t index = pTransInst->index;
+ int8_t index = pTransInst->index;
+ if (pTransInst->numOfThreads == 0) {
+ return -1;
+ }
if (pTransInst->index++ >= pTransInst->numOfThreads) {
pTransInst->index = 0;
}
@@ -946,82 +987,71 @@ int cliRBChoseIdx(STrans* pTransInst) {
}
static void doDelayTask(void* param) {
STaskArg* arg = param;
-
- SCliMsg* pMsg = arg->param1;
- SCliThrdObj* pThrd = arg->param2;
- cliHandleReq(pMsg, pThrd);
-
+ SCliMsg* pMsg = arg->param1;
+ SCliThrd* pThrd = arg->param2;
taosMemoryFree(arg);
+
+ cliHandleReq(pMsg, pThrd);
+}
+
+static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) {
+ STransConnCtx* pCtx = pMsg->ctx;
+
+ STraceId* trace = &pMsg->msg.info.traceId;
+ char tbuf[256] = {0};
+ EPSET_DEBUG_STR(&pCtx->epSet, tbuf);
+ tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf,
+ pCtx->retryCnt + 1, pCtx->retryLimit);
+
+ STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
+ arg->param1 = pMsg;
+ arg->param2 = pThrd;
+ transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+}
+
+void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
+ if (*val != exp) {
+ *val = newVal;
+ }
}
int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
- SCliThrdObj* pThrd = pConn->hostThrd;
- STrans* pTransInst = pThrd->pTransInst;
+ SCliThrd* pThrd = pConn->hostThrd;
+ STrans* pTransInst = pThrd->pTransInst;
if (pMsg == NULL || pMsg->ctx == NULL) {
tTrace("%s conn %p handle resp", pTransInst->label, pConn);
pTransInst->cfp(pTransInst->parent, pResp, NULL);
return 0;
}
-
- STransConnCtx* pCtx = pMsg->ctx;
- SEpSet* pEpSet = &pCtx->epSet;
-
- if (pCtx->retryCount == 0) {
- pCtx->origEpSet = pCtx->epSet;
- }
/*
- * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL
+ * no retry
+ * 1. query conn
+ * 2. rpc thread already receive quit msg
*/
- tmsg_t msgType = pCtx->msgType;
- if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) ||
- (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY ||
- pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) {
+ STransConnCtx* pCtx = pMsg->ctx;
+ int32_t code = pResp->code;
+ if (pTransInst->retry != NULL && pTransInst->retry(code)) {
pMsg->sent = 0;
- tTrace("try to send req to next node");
- pMsg->st = taosGetTimestampUs();
- pCtx->retryCount += 1;
- if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL && pCtx->setMaxRetry == false) {
- if (pCtx->retryCount < pEpSet->numOfEps * 3) {
- pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
- if (pThrd->quit == false) {
- STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
- arg->param1 = pMsg;
- arg->param2 = pThrd;
- transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
- transPrintEpSet(pEpSet);
- tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
- pCtx->retryCount + 1, pEpSet->numOfEps * 3);
-
- transUnrefCliHandle(pConn);
- return -1;
- }
+ pCtx->retryCnt += 1;
+ if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+ cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3);
+ if (pCtx->retryCnt < pCtx->retryLimit) {
+ transUnrefCliHandle(pConn);
+ EPSET_FORWARD_INUSE(&pCtx->epSet);
+ cliSchedMsgToNextNode(pMsg, pThrd);
+ return -1;
}
- } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
- pCtx->setMaxRetry = true;
- if (pResp->contLen == 0) {
- pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
- transPrintEpSet(&pCtx->epSet);
- tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
- pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
- } else {
- SEpSet epSet = {0};
- tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet);
- pCtx->epSet = epSet;
-
- transPrintEpSet(&pCtx->epSet);
- tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
- pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
- }
- if (pThrd->quit == false) {
- if (pResp->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) {
- if (pConn->status != ConnInPool) addConnToPool(pThrd->pool, pConn);
+ } else {
+ cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT);
+ if (pCtx->retryCnt < pCtx->retryLimit) {
+ addConnToPool(pThrd->pool, pConn);
+ if (pResp->contLen == 0) {
+ EPSET_FORWARD_INUSE(&pCtx->epSet);
} else {
- transUnrefCliHandle(pConn);
+ tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet);
}
- STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
- arg->param1 = pMsg;
- arg->param2 = pThrd;
- transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+ transFreeMsg(pResp->pCont);
+ cliSchedMsgToNextNode(pMsg, pThrd);
return -1;
}
}
@@ -1029,20 +1059,20 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
STraceId* trace = &pResp->info.traceId;
if (pCtx->pSem != NULL) {
- tGTrace("conn %p(sync) handle resp", pConn);
+ tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn);
if (pCtx->pRsp == NULL) {
- tGTrace("conn %p(sync) failed to resp, ignore", pConn);
+ tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn);
} else {
memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp));
}
tsem_post(pCtx->pSem);
pCtx->pRsp = NULL;
} else {
- tGTrace("conn %p handle resp", pConn);
- if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) {
+ tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn);
+ if (!cliIsEpsetUpdated(code, pCtx)) {
pTransInst->cfp(pTransInst->parent, pResp, NULL);
} else {
- pTransInst->cfp(pTransInst->parent, pResp, pEpSet);
+ pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet);
}
}
return 0;
@@ -1074,38 +1104,58 @@ void transUnrefCliHandle(void* handle) {
return;
}
int ref = T_REF_DEC((SCliConn*)handle);
- tTrace("%s conn %p ref %d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref);
+ tTrace("%s conn %p ref:%d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref);
if (ref == 0) {
cliDestroyConn((SCliConn*)handle, true);
}
}
+SCliThrd* transGetWorkThrdFromHandle(int64_t handle) {
+ SCliThrd* pThrd = NULL;
+ SExHandle* exh = transAcquireExHandle(refMgt, handle);
+ if (exh == NULL) {
+ return NULL;
+ }
+ pThrd = exh->pThrd;
+ transReleaseExHandle(refMgt, handle);
+ return pThrd;
+}
+SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) {
+ if (handle == 0) {
+ int idx = cliRBChoseIdx(trans);
+ if (idx < 0) return NULL;
+ return ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
+ }
+ return transGetWorkThrdFromHandle(handle);
+}
void transReleaseCliHandle(void* handle) {
- SCliThrdObj* thrd = CONN_GET_HOST_THREAD(handle);
- if (thrd == NULL) {
+ int idx = -1;
+ SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle);
+ if (pThrd == NULL) {
return;
}
-
STransMsg tmsg = {.info.handle = handle};
SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cmsg->msg = tmsg;
cmsg->type = Release;
- transSendAsync(thrd->asyncPool, &cmsg->q);
+ transSendAsync(pThrd->asyncPool, &cmsg->q);
+ return;
}
void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) {
- STrans* pTransInst = (STrans*)shandle;
- int idx = CONN_HOST_THREAD_IDX((SCliConn*)pReq->info.handle);
- if (idx == -1) {
- idx = cliRBChoseIdx(pTransInst);
+ STrans* pTransInst = (STrans*)shandle;
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
+ transFreeMsg(pReq->pCont);
+ return;
}
+
TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64());
STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
pCtx->epSet = *pEpSet;
pCtx->ahandle = pReq->info.ahandle;
pCtx->msgType = pReq->msgType;
- pCtx->hThrdIdx = idx;
if (ctx != NULL) {
pCtx->appCtx = *ctx;
@@ -1118,19 +1168,19 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra
cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
- SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
-
STraceId* trace = &pReq->info.traceId;
- tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid,
+ tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
- ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0);
+ ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0);
+ return;
}
void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) {
- STrans* pTransInst = (STrans*)shandle;
- int idx = CONN_HOST_THREAD_IDX(pReq->info.handle);
- if (idx == -1) {
- idx = cliRBChoseIdx(pTransInst);
+ STrans* pTransInst = (STrans*)shandle;
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
+ transFreeMsg(pReq->pCont);
+ return;
}
tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t));
tsem_init(sem, 0, 0);
@@ -1139,9 +1189,9 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
pCtx->epSet = *pEpSet;
+ pCtx->origEpSet = *pEpSet;
pCtx->ahandle = pReq->info.ahandle;
pCtx->msgType = pReq->msgType;
- pCtx->hThrdIdx = idx;
pCtx->pSem = sem;
pCtx->pRsp = pRsp;
@@ -1151,22 +1201,21 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
- SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
-
STraceId* trace = &pReq->info.traceId;
- tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid,
+ tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
- transSendAsync(thrd->asyncPool, &(cliMsg->q));
+ transSendAsync(pThrd->asyncPool, &(cliMsg->q));
tsem_wait(sem);
tsem_destroy(sem);
taosMemoryFree(sem);
+ return;
}
/*
*
**/
-void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) {
- STrans* pTransInst = ahandle;
+void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) {
+ STrans* pTransInst = shandle;
SCvtAddr cvtAddr = {0};
if (ip != NULL && fqdn != NULL) {
@@ -1176,14 +1225,13 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) {
}
for (int i = 0; i < pTransInst->numOfThreads; i++) {
STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
- pCtx->hThrdIdx = i;
pCtx->cvtAddr = cvtAddr;
SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cliMsg->ctx = pCtx;
cliMsg->type = Update;
- SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i];
+ SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i];
tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid);
transSendAsync(thrd->asyncPool, &(cliMsg->q));
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index 8cd7f9d827..bff7d79bd3 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -455,16 +455,16 @@ void transPrintEpSet(SEpSet* pEpSet) {
return;
}
char buf[512] = {0};
- int len = snprintf(buf, sizeof(buf), "epset { ");
+ int len = snprintf(buf, sizeof(buf), "epset:{");
for (int i = 0; i < pEpSet->numOfEps; i++) {
if (i == pEpSet->numOfEps - 1) {
- len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
+ len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
} else {
len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
}
len += snprintf(buf + len, sizeof(buf) - len, "}");
- tTrace("%s, inUse: %d", buf, pEpSet->inUse);
+ tTrace("%s, inUse:%d", buf, pEpSet->inUse);
}
bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) {
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 593a790a21..892d32696e 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -65,7 +65,7 @@ typedef struct SSvrMsg {
STransMsgType type;
} SSvrMsg;
-typedef struct SWorkThrdObj {
+typedef struct SWorkThrd {
TdThread thread;
uv_connect_t connect_req;
uv_pipe_t* pipe;
@@ -78,7 +78,7 @@ typedef struct SWorkThrdObj {
queue conn;
void* pTransInst;
bool quit;
-} SWorkThrdObj;
+} SWorkThrd;
typedef struct SServerObj {
TdThread thread;
@@ -86,10 +86,10 @@ typedef struct SServerObj {
uv_loop_t* loop;
// work thread info
- int workerIdx;
- int numOfThreads;
- int numOfWorkerReady;
- SWorkThrdObj** pThreadObj;
+ int workerIdx;
+ int numOfThreads;
+ int numOfWorkerReady;
+ SWorkThrd** pThreadObj;
uv_pipe_t pipeListen;
uv_pipe_t** pipe;
@@ -133,14 +133,14 @@ static SSvrConn* createConn(void* hThrd);
static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
static void destroyConnRegArg(SSvrConn* conn);
-static int reallocConnRefHandle(SSvrConn* conn);
+static int reallocConnRef(SSvrConn* conn);
-static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd);
-static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
- uvHandleRegister, NULL};
+static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd);
+static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd);
+static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd);
+static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd);
+static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
+ uvHandleRegister, NULL};
static int32_t exHandlesMgt;
@@ -160,7 +160,7 @@ static void* transWorkerThread(void* arg);
static void* transAcceptThread(void* arg);
// add handle loop
-static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName);
+static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName);
static bool addHandleToAcceptloop(void* arg);
#define CONN_SHOULD_RELEASE(conn, head) \
@@ -176,7 +176,7 @@ static bool addHandleToAcceptloop(void* arg);
srvMsg->msg = tmsg; \
srvMsg->type = Release; \
srvMsg->pConn = conn; \
- reallocConnRefHandle(conn); \
+ reallocConnRef(conn); \
if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \
return; \
} \
@@ -206,32 +206,6 @@ static bool addHandleToAcceptloop(void* arg);
} \
} while (0)
-#define ASYNC_CHECK_HANDLE(exh1, refId) \
- do { \
- if (refId > 0) { \
- tTrace("handle step1"); \
- SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \
- if (exh2 == NULL || refId != exh2->refId) { \
- tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
- exh2 ? exh2->refId : 0, refId); \
- goto _return1; \
- } \
- } else if (refId == 0) { \
- tTrace("handle step2"); \
- SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \
- if (exh2 == NULL || refId != exh2->refId) { \
- tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \
- exh2 ? exh2->refId : 0); \
- goto _return1; \
- } else { \
- refId = exh1->refId; \
- } \
- } else if (refId < 0) { \
- tTrace("handle step3"); \
- goto _return2; \
- } \
- } while (0)
-
void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SSvrConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
@@ -259,7 +233,7 @@ static void uvHandleReq(SSvrConn* pConn) {
// wreq->data = pConn;
// uv_read_stop((uv_stream_t*)pConn->pTcp);
// transRefSrvHandle(pConn);
- // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask);
+ // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask);
CONN_SHOULD_RELEASE(pConn, pHead);
@@ -379,7 +353,7 @@ void uvOnSendCb(uv_write_t* req, int status) {
// if (msg->type == Release && conn->status != ConnNormal) {
// conn->status = ConnNormal;
// transUnrefSrvHandle(conn);
- // reallocConnRefHandle(conn);
+ // reallocConnRef(conn);
// destroySmsg(msg);
// transQueueClear(&conn->srvMsgs);
// return;
@@ -448,6 +422,8 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
transUnrefSrvHandle(pConn);
} else {
pHead->msgType = pMsg->msgType;
+ if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead))
+ pHead->msgType = pConn->inType + 1;
}
}
@@ -504,7 +480,7 @@ static void destroySmsg(SSvrMsg* smsg) {
transFreeMsg(smsg->msg.pCont);
taosMemoryFree(smsg);
}
-static void destroyAllConn(SWorkThrdObj* pThrd) {
+static void destroyAllConn(SWorkThrd* pThrd) {
tTrace("thread %p destroy all conn ", pThrd);
while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
queue* h = QUEUE_HEAD(&pThrd->conn);
@@ -519,10 +495,10 @@ static void destroyAllConn(SWorkThrdObj* pThrd) {
}
}
void uvWorkerAsyncCb(uv_async_t* handle) {
- SAsyncItem* item = handle->data;
- SWorkThrdObj* pThrd = item->pThrd;
- SSvrConn* conn = NULL;
- queue wq;
+ SAsyncItem* item = handle->data;
+ SWorkThrd* pThrd = item->pThrd;
+ SSvrConn* conn = NULL;
+ queue wq;
// batch process to avoid to lock/unlock frequently
taosThreadMutexLock(&item->mtx);
@@ -650,7 +626,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
assert(buf->base[0] == notify[0]);
taosMemoryFree(buf->base);
- SWorkThrdObj* pThrd = q->data;
+ SWorkThrd* pThrd = q->data;
uv_pipe_t* pipe = (uv_pipe_t*)q;
if (!uv_pipe_pending_count(pipe)) {
@@ -718,10 +694,10 @@ void uvOnPipeConnectionCb(uv_connect_t* connect, int status) {
if (status != 0) {
return;
}
- SWorkThrdObj* pThrd = container_of(connect, SWorkThrdObj, connect_req);
+ SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req);
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
}
-static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName) {
+static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) {
pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t));
if (0 != uv_loop_init(pThrd->loop)) {
return false;
@@ -774,14 +750,14 @@ static bool addHandleToAcceptloop(void* arg) {
}
void* transWorkerThread(void* arg) {
setThreadName("trans-worker");
- SWorkThrdObj* pThrd = (SWorkThrdObj*)arg;
+ SWorkThrd* pThrd = (SWorkThrd*)arg;
uv_run(pThrd->loop, UV_RUN_DEFAULT);
return NULL;
}
static SSvrConn* createConn(void* hThrd) {
- SWorkThrdObj* pThrd = hThrd;
+ SWorkThrd* pThrd = hThrd;
SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn));
QUEUE_INIT(&pConn->queue);
@@ -826,7 +802,7 @@ static void destroyConnRegArg(SSvrConn* conn) {
conn->regArg.init = 0;
}
}
-static int reallocConnRefHandle(SSvrConn* conn) {
+static int reallocConnRef(SSvrConn* conn) {
transReleaseExHandle(refMgt, conn->refId);
transRemoveExHandle(refMgt, conn->refId);
// avoid app continue to send msg on invalid handle
@@ -844,7 +820,7 @@ static void uvDestroyConn(uv_handle_t* handle) {
if (conn == NULL) {
return;
}
- SWorkThrdObj* thrd = conn->hostThrd;
+ SWorkThrd* thrd = conn->hostThrd;
transReleaseExHandle(refMgt, conn->refId);
transRemoveExHandle(refMgt, conn->refId);
@@ -889,7 +865,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
srv->numOfThreads = numOfThreads;
srv->workerIdx = 0;
srv->numOfWorkerReady = 0;
- srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*));
+ srv->pThreadObj = (SWorkThrd**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd*));
srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*));
srv->ip = ip;
srv->port = port;
@@ -914,7 +890,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
assert(0 == uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb));
for (int i = 0; i < srv->numOfThreads; i++) {
- SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj));
+ SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd));
thrd->pTransInst = shandle;
thrd->quit = false;
srv->pThreadObj[i] = thrd;
@@ -959,7 +935,7 @@ End:
return NULL;
}
-void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd) {
thrd->quit = true;
if (QUEUE_IS_EMPTY(&thrd->conn)) {
uv_walk(thrd->loop, uvWalkCb, NULL);
@@ -968,10 +944,10 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) {
}
taosMemoryFree(msg);
}
-void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd) {
SSvrConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
- reallocConnRefHandle(conn);
+ reallocConnRef(conn);
if (!transQueuePush(&conn->srvMsgs, msg)) {
return;
}
@@ -982,12 +958,12 @@ void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) {
}
destroySmsg(msg);
}
-void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd) {
// send msg to client
tDebug("%s conn %p start to send resp (2/2)", transLabel(thrd->pTransInst), msg->pConn);
uvStartSendResp(msg);
}
-void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) {
+void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) {
SSvrConn* conn = msg->pConn;
tDebug("%s conn %p register brokenlink callback", transLabel(thrd->pTransInst), conn);
if (conn->status == ConnAcquire) {
@@ -1008,7 +984,7 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) {
taosMemoryFree(msg);
}
}
-void destroyWorkThrd(SWorkThrdObj* pThrd) {
+void destroyWorkThrd(SWorkThrd* pThrd) {
if (pThrd == NULL) {
return;
}
@@ -1019,7 +995,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) {
taosMemoryFree(pThrd->loop);
taosMemoryFree(pThrd);
}
-void sendQuitToWorkThrd(SWorkThrdObj* pThrd) {
+void sendQuitToWorkThrd(SWorkThrd* pThrd) {
SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg));
msg->type = Quit;
tDebug("server send quit msg to work thread");
@@ -1055,8 +1031,6 @@ void transCloseServer(void* arg) {
int ref = atomic_sub_fetch_32(&tranSSvrInst, 1);
if (ref == 0) {
- // TdThreadOnce tmpInit = PTHREAD_ONCE_INIT;
- // memcpy(&transModuleInit, &tmpInit, sizeof(TdThreadOnce));
transCloseExHandleMgt(refMgt);
}
}
@@ -1086,7 +1060,7 @@ void transReleaseSrvHandle(void* handle) {
ASYNC_CHECK_HANDLE(exh, refId);
- SWorkThrdObj* pThrd = exh->pThrd;
+ SWorkThrd* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
STransMsg tmsg = {.code = 0, .info.handle = exh, .info.ahandle = NULL, .info.refId = refId};
@@ -1116,7 +1090,7 @@ void transSendResponse(const STransMsg* msg) {
STransMsg tmsg = *msg;
tmsg.info.refId = refId;
- SWorkThrdObj* pThrd = exh->pThrd;
+ SWorkThrd* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg));
@@ -1146,7 +1120,7 @@ void transRegisterMsg(const STransMsg* msg) {
STransMsg tmsg = *msg;
tmsg.info.refId = refId;
- SWorkThrdObj* pThrd = exh->pThrd;
+ SWorkThrd* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg));
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 682afbb785..2de0fea9ac 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -104,6 +104,8 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
return -1;
}
+ pRead->pReadLogTFile = pLogTFile;
+
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxTFile == NULL) {
@@ -112,7 +114,6 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
return -1;
}
- pRead->pReadLogTFile = pLogTFile;
pRead->pReadIdxTFile = pIdxTFile;
return 0;
}
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index 1d169a0891..d1295667af 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -42,6 +42,9 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
taosRemoveFile(fnameStr);
+
+ walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
+ taosRemoveFile(fnameStr);
}
}
walRemoveMeta(pWal);
@@ -105,7 +108,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
- TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
+ TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
taosThreadMutexUnlock(&pWal->mutex);
@@ -126,7 +129,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT(entry.ver == ver);
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
- TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
+ TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pLogTFile == NULL) {
// TODO
taosThreadMutexUnlock(&pWal->mutex);
@@ -307,8 +310,8 @@ int walRoll(SWal *pWal) {
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
SWalIdxEntry entry = {.ver = ver, .offset = offset};
- /*int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_CUR);*/
- /*wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);*/
+ int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END);
+ wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);
int64_t size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
index 05b7498cc0..291d38bdca 100644
--- a/source/os/src/osFile.c
+++ b/source/os/src/osFile.c
@@ -157,14 +157,14 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
#ifdef WINDOWS
bool code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
if (!code) {
- printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+ printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno));
}
return !code;
#else
int32_t code = rename(oldName, newName);
if (code < 0) {
- printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+ printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno));
}
return code;
@@ -671,7 +671,7 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) {
fflush(pFile->fp);
}
-bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; }
+bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; }
int32_t taosUmaskFile(int32_t maskVal) {
#ifdef WINDOWS
diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index 4d61e7036d..b0e07ff010 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -946,7 +946,7 @@ int32_t taosGetFqdn(char *fqdn) {
#endif // __APPLE__
int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (!result) {
- fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
+ fprintf(stderr, "failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
return -1;
}
@@ -1073,7 +1073,7 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) {
* Set TCP connection timeout per-socket level.
* ref [https://github.com/libuv/help/issues/54]
*/
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) {
+int32_t taosCreateSocketWithTimeout(uint32_t timeout) {
#if defined(WINDOWS)
SOCKET fd;
#else
@@ -1083,11 +1083,11 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) {
return -1;
}
#if defined(WINDOWS)
- if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) {
+ if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) {
return -1;
}
#else // Linux like systems
- uint32_t conn_timeout_ms = conn_timeout_sec * 1000;
+ uint32_t conn_timeout_ms = timeout * 1000;
if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {
return -1;
}
diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh
index 3f23cd8b5f..f80c2b217f 100755
--- a/tests/parallel_test/container_build.sh
+++ b/tests/parallel_test/container_build.sh
@@ -52,7 +52,7 @@ fi
docker run \
-v $REP_MOUNT_PARAM \
- --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
+ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
ret=$?
exit $ret
diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py
index 6eb78f0771..21cad4c5bb 100644
--- a/tests/pytest/util/cluster.py
+++ b/tests/pytest/util/cluster.py
@@ -24,7 +24,7 @@ class ClusterDnodes(TDDnodes):
class ConfigureyCluster:
- """This will create defined number of dnodes and create a cluset.
+ """This will create defined number of dnodes and create a cluster.
at the same time, it will return TDDnodes list: dnodes, """
hostname= socket.gethostname()
@@ -85,8 +85,8 @@ class ConfigureyCluster:
count+=1
time.sleep(1)
else:
- tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
- return -1
+ tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
+
cluster = ConfigureyCluster()
\ No newline at end of file
diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py
index e5095c74dc..807398f70f 100644
--- a/tests/pytest/util/constant.py
+++ b/tests/pytest/util/constant.py
@@ -1,5 +1,76 @@
# -*- coding: utf-8 -*-
+# basic type
+TAOS_DATA_TYPE = [
+ "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED",
+ "FLOAT", "DOUBLE",
+ "BOOL",
+ "BINARY", "NCHAR", "VARCHAR",
+ "TIMESTAMP",
+ # "MEDIUMBLOB", "BLOB", # add in 3.x
+ # "DECIMAL", "NUMERIC", # add in 3.x
+ "JSON", # only for tag
+]
+
+TAOS_NUM_TYPE = [
+ "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", "FLOAT", "DOUBLE",
+ # "DECIMAL", "NUMERIC", # add in 3.x
+]
+TAOS_CHAR_TYPE = [
+ "BINARY", "NCHAR", "VARCHAR",
+]
+TAOS_BOOL_TYPE = ["BOOL",]
+TAOS_TS_TYPE = ["TIMESTAMP",]
+TAOS_BIN_TYPE = [
+ "MEDIUMBLOB", "BLOB", # add in 3.x
+]
+
+TAOS_TIME_INIT = ["b", "u", "a", "s", "m", "h", "d", "w", "n", "y"]
+TAOS_PRECISION = ["ms", "us", "ns"]
+PRECISION_DEFAULT = "ms"
+PRECISION = PRECISION_DEFAULT
+
+TAOS_KEYWORDS = [
+ "ABORT", "CREATE", "IGNORE", "NULL", "STAR",
+ "ACCOUNT", "CTIME", "IMMEDIATE", "OF", "STATE",
+ "ACCOUNTS", "DATABASE", "IMPORT", "OFFSET", "STATEMENT",
+ "ADD", "DATABASES", "IN", "OR", "STATE_WINDOW",
+ "AFTER", "DAYS", "INITIALLY", "ORDER", "STORAGE",
+ "ALL", "DBS", "INSERT", "PARTITIONS", "STREAM",
+ "ALTER", "DEFERRED", "INSTEAD", "PASS", "STREAMS",
+ "AND", "DELIMITERS", "INT", "PLUS", "STRING",
+ "AS", "DESC", "INTEGER", "PPS", "SYNCDB",
+ "ASC", "DESCRIBE", "INTERVAL", "PRECISION", "TABLE",
+ "ATTACH", "DETACH", "INTO", "PREV", "TABLES",
+ "BEFORE", "DISTINCT", "IS", "PRIVILEGE", "TAG",
+ "BEGIN", "DIVIDE", "ISNULL", "QTIME", "TAGS",
+ "BETWEEN", "DNODE", "JOIN", "QUERIES", "TBNAME",
+ "BIGINT", "DNODES", "KEEP", "QUERY", "TIMES",
+ "BINARY", "DOT", "KEY", "QUORUM", "TIMESTAMP",
+ "BITAND", "DOUBLE", "KILL", "RAISE", "TINYINT",
+ "BITNOT", "DROP", "LE", "REM", "TOPIC",
+ "BITOR", "EACH", "LIKE", "REPLACE", "TOPICS",
+ "BLOCKS", "END", "LIMIT", "REPLICA", "TRIGGER",
+ "BOOL", "EQ", "LINEAR", "RESET", "TSERIES",
+ "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS",
+ "CACHE", "EXPLAIN", "LP", "ROW", "UNION",
+ "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED",
+ "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE",
+ "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS",
+ "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE",
+ "COLON", "FOR", "MINROWS", "SEMI", "USER",
+ "COLUMN", "FROM", "MINUS", "SESSION", "USERS",
+ "COMMA", "FSYNC", "MNODES", "SET", "USING",
+ "COMP", "GE", "MODIFY", "SHOW", "VALUES",
+ "COMPACT", "GLOB", "MODULES", "SLASH", "VARIABLE",
+ "CONCAT", "GRANTS", "NCHAR", "SLIDING", "VARIABLES",
+ "CONFLICT", "GROUP", "NE", "SLIMIT", "VGROUPS",
+ "CONNECTION", "GT", "NONE", "SMALLINT", "VIEW",
+ "CONNECTIONS", "HAVING", "NOT", "SOFFSET", "VNODES",
+ "CONNS", "ID", "NOTNULL", "STABLE", "WAL",
+ "COPY", "IF", "NOW", "STABLES", "WHERE",
+]
+
# basic data type boundary
TINYINT_MAX = 127
TINYINT_MIN = -128
@@ -11,7 +82,7 @@ SMALLINT_MAX = 32767
SMALLINT_MIN = -32768
SMALLINT_UN_MAX = 65535
-MALLINT_UN_MIN = 0
+SMALLINT_UN_MIN = 0
INT_MAX = 2147483647
INT_MIN = -2147483648
@@ -33,8 +104,8 @@ DOUBLE_MIN = -1.7E+308
# schema boundary
BINARY_LENGTH_MAX = 16374
-NCAHR_LENGTH_MAX_ = 4093
-DBNAME_LENGTH_MAX_ = 64
+NCAHR_LENGTH_MAX = 4093
+DBNAME_LENGTH_MAX = 64
STBNAME_LENGTH_MAX = 192
STBNAME_LENGTH_MIN = 1
@@ -66,4 +137,32 @@ MNODE_SHM_SIZE_DEFAULT = 6292480
VNODE_SHM_SIZE_MAX = 2147483647
VNODE_SHM_SIZE_MIN = 6292480
-VNODE_SHM_SIZE_DEFAULT = 31458304
\ No newline at end of file
+VNODE_SHM_SIZE_DEFAULT = 31458304
+
+# time_init
+TIME_MS = 1
+TIME_US = TIME_MS/1000
+TIME_NS = TIME_US/1000
+
+TIME_S = 1000 * TIME_MS
+TIME_M = 60 * TIME_S
+TIME_H = 60 * TIME_M
+TIME_D = 24 * TIME_H
+TIME_W = 7 * TIME_D
+TIME_N = 30 * TIME_D
+TIME_Y = 365 * TIME_D
+
+
+# session parameters
+INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US
+
+
+# streams and related agg-function
+SMA_INDEX_FUNCTIONS = ["MIN", "MAX"]
+ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"]
+SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"]
+WATERMARK_MAX = 900000
+WATERMARK_MIN = 0
+
+MAX_DELAY_MAX = 900000
+MAX_DELAY_MIN = 1
\ No newline at end of file
diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py
index be3454f78f..a38b14a52d 100644
--- a/tests/pytest/util/dnodes.py
+++ b/tests/pytest/util/dnodes.py
@@ -508,7 +508,6 @@ class TDDnode:
def stoptaosd(self):
if (not self.remoteIP == ""):
- print("123")
self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
tdLog.info("stop dnode%d"%self.index)
return
@@ -518,18 +517,21 @@ class TDDnode:
toBeKilled = "valgrind.bin"
if self.running != 0:
- psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index)
- processID = subprocess.check_output(
- psCmd, shell=True).decode("utf-8")
-
- while(processID):
- killCmd = "kill -INT %s > /dev/null 2>&1" % processID
- os.system(killCmd)
- time.sleep(1)
+ if platform.system().lower() == 'windows':
+ os.system("wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId | xargs echo | awk '{print $2}' | xargs taskkill -f -pid"%self.index)
+ else:
+ psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
- if self.valgrind:
- time.sleep(2)
+
+ while(processID):
+ killCmd = "kill -INT %s > /dev/null 2>&1" % processID
+ os.system(killCmd)
+ time.sleep(1)
+ processID = subprocess.check_output(
+ psCmd, shell=True).decode("utf-8")
+ if self.valgrind:
+ time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index 580fc8ee47..fe802dd9a3 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -21,6 +21,7 @@ import psutil
import shutil
import pandas as pd
from util.log import *
+from util.constant import *
def _parse_datetime(timestr):
try:
@@ -117,8 +118,7 @@ class TDSql:
col_name_list = []
col_type_list = []
self.cursor.execute(sql)
- self.queryCols = self.cursor.description
- for query_col in self.queryCols:
+ for query_col in self.cursor.description:
col_name_list.append(query_col[0])
col_type_list.append(query_col[1])
except Exception as e:
@@ -301,6 +301,41 @@ class TDSql:
args = (caller.filename, caller.lineno, self.sql, elm, expect_elm)
tdLog.exit("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args)
+ def get_times(self, time_str, precision="ms"):
+ caller = inspect.getframeinfo(inspect.stack()[1][0])
+ if time_str[-1] not in TAOS_TIME_INIT:
+ tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {time_str} not a standard taos time init")
+ if precision not in TAOS_PRECISION:
+ tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {precision} not a standard taos time precision")
+
+ if time_str[-1] == TAOS_TIME_INIT[0]:
+ times = int(time_str[:-1]) * TIME_NS
+ if time_str[-1] == TAOS_TIME_INIT[1]:
+ times = int(time_str[:-1]) * TIME_US
+ if time_str[-1] == TAOS_TIME_INIT[2]:
+ times = int(time_str[:-1]) * TIME_MS
+ if time_str[-1] == TAOS_TIME_INIT[3]:
+ times = int(time_str[:-1]) * TIME_S
+ if time_str[-1] == TAOS_TIME_INIT[4]:
+ times = int(time_str[:-1]) * TIME_M
+ if time_str[-1] == TAOS_TIME_INIT[5]:
+ times = int(time_str[:-1]) * TIME_H
+ if time_str[-1] == TAOS_TIME_INIT[6]:
+ times = int(time_str[:-1]) * TIME_D
+ if time_str[-1] == TAOS_TIME_INIT[7]:
+ times = int(time_str[:-1]) * TIME_W
+ if time_str[-1] == TAOS_TIME_INIT[8]:
+ times = int(time_str[:-1]) * TIME_N
+ if time_str[-1] == TAOS_TIME_INIT[9]:
+ times = int(time_str[:-1]) * TIME_Y
+
+ if precision == "ms":
+ return int(times)
+ elif precision == "us":
+ return int(times*1000)
+ elif precision == "ns":
+ return int(times*1000*1000)
+
def taosdStatus(self, state):
tdLog.sleep(5)
pstate = 0
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 1b61bd1890..181342b4b7 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -29,8 +29,11 @@
#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
-#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
# ---- insert
./test.sh -f tsim/insert/basic0.sim
diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat
new file mode 100644
index 0000000000..81c1f21c04
--- /dev/null
+++ b/tests/script/test-all.bat
@@ -0,0 +1,68 @@
+@echo off
+SETLOCAL EnableDelayedExpansion
+for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a")
+set /a a=0
+echo Windows Taosd Full Test
+set /a exitNum=0
+rm -rf failed.txt
+set caseFile="jenkins\\basic.txt"
+if not "%2" == "" (
+ set caseFile="%2"
+)
+for /F "usebackq tokens=*" %%i in (!caseFile!) do (
+ set line=%%i
+ call :CheckSkipCase %%i
+ if !skipCase! == false (
+ if "!line:~,9!" == "./test.sh" (
+ set /a a+=1
+ echo !a! Processing %%i
+ call :GetTimeSeconds !time!
+ set time1=!_timeTemp!
+ echo Start at !time!
+ call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
+ if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+ )
+ )
+)
+exit !exitNum!
+
+:colorEcho
+set timeNow=%time%
+call :GetTimeSeconds %timeNow%
+set time2=%_timeTemp%
+set /a interTime=%time2% - %time1%
+echo End at %timeNow% , cast %interTime%s
+echo off
+ "%~2"
+findstr /v /a:%1 /R "^$" "%~2" nul
+del "%~2" > nul 2>&1i
+goto :eof
+
+:GetTimeSeconds
+set tt=%1
+set tt=%tt:.= %
+set tt=%tt::= %
+set tt=%tt: 0= %
+set /a index=1
+for %%a in (%tt%) do (
+ if !index! EQU 1 (
+ set /a hh=%%a
+ )^
+ else if !index! EQU 2 (
+ set /a mm=%%a
+
+ )^
+ else if !index! EQU 3 (
+ set /a ss=%%a
+ )
+ set /a index=index+1
+)
+set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
+goto :eof
+
+:CheckSkipCase
+set skipCase=false
+if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
+if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true )
+if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true )
+:goto eof
\ No newline at end of file
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim
new file mode 100644
index 0000000000..d3b5a02a23
--- /dev/null
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim
@@ -0,0 +1,178 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+sql create user u1 pass 'taosdata'
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+
+$x = 0
+step1:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 4 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step1
+endi
+if $data(2)[4] != ready then
+ goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 1 replica 1
+
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+$x = 0
+step2:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 4 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step2
+endi
+if $data(2)[4] != ready then
+ goto step2
+endi
+if $data(3)[4] != ready then
+ goto step2
+endi
+if $data(4)[4] != ready then
+ goto step2
+endi
+
+print =============== step3: create database
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step40:
+print redistribute vgroup 2 dnode 3
+sql redistribute vgroup 2 dnode 3
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step41:
+print redistribute vgroup 2 dnode 4
+sql redistribute vgroup 2 dnode 4
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step42:
+print redistribute vgroup 2 dnode 2
+sql redistribute vgroup 2 dnode 2
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step43:
+print redistribute vgroup 2 dnode 3
+sql redistribute vgroup 2 dnode 3
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step44:
+print redistribute vgroup 2 dnode 4
+sql redistribute vgroup 2 dnode 4
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step45:
+print redistribute vgroup 2 dnode 2
+sql redistribute vgroup 2 dnode 2
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step46:
+print redistribute vgroup 2 dnode 3
+sql redistribute vgroup 2 dnode 3
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step47:
+print redistribute vgroup 2 dnode 2
+sql redistribute vgroup 2 dnode 2
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
index 9b11e16616..00c6e8a98d 100644
--- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
@@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4
# vgroup not exist
sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4
# un changed
-sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4
+# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4
# no enought vnodes
sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4
# offline vnodes
@@ -176,8 +176,6 @@ if $rows != 1 then
return -1
endi
-return
-
print =============== step32:
print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5
sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim
new file mode 100644
index 0000000000..6614009497
--- /dev/null
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim
@@ -0,0 +1,245 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/deploy.sh -n dnode6 -i 6
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+sql create user u1 pass 'taosdata'
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+sql create dnode $hostname port 7600
+
+$x = 0
+step1:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 6 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step1
+endi
+if $data(2)[4] != ready then
+ goto step1
+endi
+if $data(3)[4] != ready then
+ goto step1
+endi
+if $data(4)[4] != ready then
+ goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 1 replica 3
+
+system sh/exec.sh -n dnode5 -s start
+system sh/exec.sh -n dnode6 -s start
+$x = 0
+step2:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 6 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step2
+endi
+if $data(2)[4] != ready then
+ goto step2
+endi
+if $data(3)[4] != ready then
+ goto step2
+endi
+if $data(4)[4] != ready then
+ goto step2
+endi
+if $data(5)[4] != ready then
+ goto step2
+endi
+if $data(6)[4] != ready then
+ goto step2
+endi
+
+print =============== step3: create database
+$leaderExist = 0
+$leaderVnode = 0
+$follower1 = 0
+$follower2 = 0
+
+$x = 0
+step3:
+ $x = $x + 1
+ sleep 1000
+ if $x == 60 then
+ print ====> db not ready!
+ return -1
+ endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $rows != 1 then
+ return -1
+endi
+if $data(2)[4] == leader then
+ $leaderExist = 1
+ $leaderVnode = 2
+ $follower1 = 3
+ $follower2 = 4
+endi
+if $data(2)[6] == leader then
+ $leaderExist = 1
+ $leaderVnode = 3
+ $follower1 = 2
+ $follower2 = 4
+endi
+if $data(2)[8] == leader then
+ $leaderExist = 1
+ $leaderVnode = 4
+ $follower1 = 2
+ $follower2 = 3
+endi
+if $leaderExist != 1 then
+ goto step3
+endi
+
+print leader $leaderVnode
+print follower1 $follower1
+print follower2 $follower2
+
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step40:
+print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6
+sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step41:
+print redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1
+sql redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step42:
+print redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode
+sql redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step43:
+print redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6
+sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step44:
+print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode
+sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step45:
+print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6
+sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step46:
+print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
+sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step47:
+print redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode
+sql redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGINT
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim
new file mode 100644
index 0000000000..c78d90e352
--- /dev/null
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim
@@ -0,0 +1,263 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/deploy.sh -n dnode6 -i 6
+system sh/deploy.sh -n dnode7 -i 7
+system sh/deploy.sh -n dnode8 -i 8
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode7 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode8 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+sql create user u1 pass 'taosdata'
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+sql create dnode $hostname port 7600
+sql create dnode $hostname port 7700
+sql create dnode $hostname port 7800
+
+$x = 0
+step1:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 8 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step1
+endi
+if $data(2)[4] != ready then
+ goto step1
+endi
+if $data(3)[4] != ready then
+ goto step1
+endi
+if $data(4)[4] != ready then
+ goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 1 replica 3
+
+system sh/exec.sh -n dnode5 -s start
+system sh/exec.sh -n dnode6 -s start
+system sh/exec.sh -n dnode7 -s start
+system sh/exec.sh -n dnode8 -s start
+$x = 0
+step2:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 8 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step2
+endi
+if $data(2)[4] != ready then
+ goto step2
+endi
+if $data(3)[4] != ready then
+ goto step2
+endi
+if $data(4)[4] != ready then
+ goto step2
+endi
+if $data(5)[4] != ready then
+ goto step2
+endi
+if $data(6)[4] != ready then
+ goto step2
+endi
+if $data(7)[4] != ready then
+ goto step2
+endi
+if $data(8)[4] != ready then
+ goto step2
+endi
+
+print =============== step3: create database
+$leaderExist = 0
+$leaderVnode = 0
+$follower1 = 0
+$follower2 = 0
+
+$x = 0
+step3:
+ $x = $x + 1
+ sleep 1000
+ if $x == 60 then
+ print ====> db not ready!
+ return -1
+ endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $rows != 1 then
+ return -1
+endi
+if $data(2)[4] == leader then
+ $leaderExist = 1
+ $leaderVnode = 2
+ $follower1 = 3
+ $follower2 = 4
+endi
+if $data(2)[6] == leader then
+ $leaderExist = 1
+ $leaderVnode = 3
+ $follower1 = 2
+ $follower2 = 4
+endi
+if $data(2)[8] == leader then
+ $leaderExist = 1
+ $leaderVnode = 4
+ $follower1 = 2
+ $follower2 = 3
+endi
+if $leaderExist != 1 then
+ goto step3
+endi
+
+print leader $leaderVnode
+print follower1 $follower1
+print follower2 $follower2
+
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step40:
+print redistribute vgroup 2 dnode 7 dnode 5 dnode 6
+sql redistribute vgroup 2 dnode 7 dnode 5 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step41:
+print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1
+sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step42:
+print redistribute vgroup 2 dnode 7 dnode 6 dnode 8
+sql redistribute vgroup 2 dnode 7 dnode 6 dnode 8
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step43:
+print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
+sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+
+return
+
+
+print =============== step44:
+print redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode
+sql redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step45:
+print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 8
+sql redistribute vgroup 2dnode $follower1 dnode $follower2 dnode 8
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step46:
+print redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5
+sql redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+print =============== step47:
+print redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8
+sql redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGINT
diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim
index 91ce49bc8c..8936655974 100644
--- a/tests/script/tsim/stream/distributeInterval0.sim
+++ b/tests/script/tsim/stream/distributeInterval0.sim
@@ -10,6 +10,30 @@ sql create dnode $hostname2 port 7200
system sh/exec.sh -n dnode2 -s start
+print ===== step1
+$x = 0
+step1:
+ $x = $x + 1
+ sleep 1000
+ if $x == 10 then
+ print ====> dnode not ready!
+ return -1
+ endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+ return -1
+endi
+if $data(1)[4] != ready then
+ goto step1
+endi
+if $data(2)[4] != ready then
+ goto step1
+endi
+
+print ===== step2
+
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
@@ -80,17 +104,17 @@ endi
if $data03 != 4 then
print ======$data03
- return -1
+ goto loop1
endi
if $data04 != 52 then
print ======$data04
- return -1
+ goto loop1
endi
if $data05 != 13 then
print ======$data05
- return -1
+ goto loop1
endi
# row 1
@@ -179,7 +203,7 @@ sql use test1;
sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
-sql create stream stream_t2 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ;
+sql create stream stream_t2 trigger at_once watermark 20s into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ;
sql insert into ts1 values(1648791211000,1,2,3);
sql insert into ts1 values(1648791222001,2,2,3);
diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim
index b84a01eb4a..c634ad85ee 100644
--- a/tests/script/tsim/stream/partitionby.sim
+++ b/tests/script/tsim/stream/partitionby.sim
@@ -74,7 +74,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
-sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ;
+sql create stream stream_t2 trigger at_once watermark 20s into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ;
sql insert into ts1 values(1648791211000,1,2,3,1);
sql insert into ts1 values(1648791222001,2,2,3,2);
sql insert into ts2 values(1648791211000,1,2,3,3);
@@ -83,16 +83,16 @@ sql insert into ts2 values(1648791222001,2,2,3,4);
sql insert into ts2 values(1648791222002,2,2,3,5);
sql insert into ts2 values(1648791222002,2,2,3,6);
-sql insert into ts1 values(1648791211000,1,2,3,1);
-sql insert into ts1 values(1648791222001,2,2,3,2);
-sql insert into ts2 values(1648791211000,1,2,3,3);
-sql insert into ts2 values(1648791222001,2,2,3,4);
+sql insert into ts1 values(1648791211000,1,2,3,7);
+sql insert into ts1 values(1648791222001,2,2,3,8);
+sql insert into ts2 values(1648791211000,1,2,3,9);
+sql insert into ts2 values(1648791222001,2,2,3,10);
$loop_count = 0
loop2:
sleep 300
-sql select * from streamtST;
+sql select * from streamtST order by c7 asc;
$loop_count = $loop_count + 1
if $loop_count == 10 then
@@ -104,8 +104,18 @@ print =====data01=$data01
goto loop2
endi
-if $data02 != 1 then
-print =====data02=$data02
+if $data11 != 1 then
+print =====data11=$data11
+goto loop2
+endi
+
+if $data21 != 1 then
+print =====data21=$data21
+goto loop2
+endi
+
+if $data31 != 2 then
+print =====data31=$data31
goto loop2
endi
@@ -114,8 +124,18 @@ print =====data03=$data03
goto loop2
endi
-if $data04 != 2 then
-print =====data04=$data04
+if $data13 != 2 then
+print =====data13=$data13
+goto loop2
+endi
+
+if $data23 != 1 then
+print =====data23=$data23
+goto loop2
+endi
+
+if $data33 != 4 then
+print =====data33=$data33
goto loop2
endi
diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim
index dbf714a96f..dbdaaf65d0 100644
--- a/tests/script/tsim/stream/schedSnode.sim
+++ b/tests/script/tsim/stream/schedSnode.sim
@@ -79,17 +79,17 @@ endi
if $data03 != 4 then
print ======$data03
- return -1
+ goto loop1
endi
if $data04 != 52 then
print ======$data04
- return -1
+ goto loop1
endi
if $data05 != 13 then
print ======$data05
- return -1
+ goto loop1
endi
# row 1
diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim
index ea3294765c..10c31dc288 100644
--- a/tests/script/tsim/user/privilege_sysinfo.sim
+++ b/tests/script/tsim/user/privilege_sysinfo.sim
@@ -8,14 +8,12 @@ sql create user sysinfo0 pass 'taosdata'
sql create user sysinfo1 pass 'taosdata'
sql alter user sysinfo0 sysinfo 0
sql alter user sysinfo1 sysinfo 1
+sql create database db
print user sysinfo0 login
sql close
sql connect sysinfo0
-system sh/exec.sh -n dnode1 -s stop
-return
-
print =============== check oper
sql_error create user u1 pass 'u1'
sql_error drop user sysinfo1
@@ -42,6 +40,24 @@ sql_error kill connection 1
sql_error kill query 1
print =============== check db
-sql_error create database db
+sql_error create database d1
+sql_error drop database db
+sql_error use db
+sql_error alter database db replica 1;
+sql_error show db.vgroups
+sql_error show db.stables
+sql_error show db.tables
+
+print =============== check show
+sql_error show users
+sql_error show cluster
+sql_error show dnodes
+sql_error show mnodes
+sql_error show snodes
+sql_error show qnodes
+sql_error show bnodes
+sql_error show grants
+sql_error show dnode 1 variables;
+sql show variables;
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py
index 5a0684e2ee..4b37eeb9a5 100644
--- a/tests/system-test/1-insert/create_retentions.py
+++ b/tests/system-test/1-insert/create_retentions.py
@@ -21,9 +21,9 @@ SINT_UN_COL = "c_sint_un"
BINT_UN_COL = "c_bint_un"
INT_UN_COL = "c_int_un"
-BINARY_COL = "c8"
-NCHAR_COL = "c9"
-TS_COL = "c10"
+BINARY_COL = "c_binary"
+NCHAR_COL = "c_nchar"
+TS_COL = "c_ts"
NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
CHAR_COL = [ BINARY_COL, NCHAR_COL, ]
@@ -51,12 +51,28 @@ class DataSet:
binary_data : List[str] = None
nchar_data : List[str] = None
+ def __post_init__(self):
+ self.ts_data = []
+ self.int_data = []
+ self.bint_data = []
+ self.sint_data = []
+ self.tint_data = []
+ self.int_un_data = []
+ self.bint_un_data = []
+ self.sint_un_data = []
+ self.tint_un_data = []
+ self.float_data = []
+ self.double_data = []
+ self.bool_data = []
+ self.binary_data = []
+ self.nchar_data = []
+
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
- tdSql.init(conn.cursor(), True)
+ tdSql.init(conn.cursor(), False)
@property
def create_databases_sql_err(self):
@@ -87,28 +103,28 @@ class TDTestCase:
@property
def create_stable_sql_err(self):
return [
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m",
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s",
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ",
+ # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ",
+ # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s",
- f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m",
- # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s",
+ f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m",
+ # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s",
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " ,
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " ,
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " ,
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " ,
- # watermark, maxdelay: [0, 900000], [ms, s, m, ?]
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u",
+ # watermark, max_delay: [0, 900000], [ms, s, m, ?]
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h",
- f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h",
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m",
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s",
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h",
+ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d",
]
@@ -117,11 +133,11 @@ class TDTestCase:
def create_stable_sql_current(self):
return [
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)",
- f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m",
- f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m",
- f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m",
- # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m",
- # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m",
+ f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m",
+ f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m",
+ f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m",
+ # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m",
+ # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m",
]
def test_create_stb(self):
@@ -135,7 +151,7 @@ class TDTestCase:
tdSql.checkRows(len(self.create_stable_sql_current))
# tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database
- # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0")
+ # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m")
def test_create_databases(self):
@@ -177,21 +193,6 @@ class TDTestCase:
def __data_set(self, rows):
data_set = DataSet()
- # neg_data_set = DataSet()
- data_set.ts_data = []
- data_set.int_data = []
- data_set.bint_data = []
- data_set.sint_data = []
- data_set.tint_data = []
- data_set.int_un_data = []
- data_set.bint_un_data = []
- data_set.sint_un_data = []
- data_set.tint_un_data = []
- data_set.float_data = []
- data_set.double_data = []
- data_set.bool_data = []
- data_set.binary_data = []
- data_set.nchar_data = []
for i in range(rows):
data_set.ts_data.append(NOW + 1 * (rows - i))
@@ -226,6 +227,7 @@ class TDTestCase:
return data_set
def __insert_data(self):
+ tdLog.printNoPrefix("==========step: start inser data into tables now.....")
data = self.__data_set(rows=self.rows)
# now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
@@ -264,10 +266,10 @@ class TDTestCase:
def run(self):
self.rows = 10
-
+ tdSql.prepare()
tdLog.printNoPrefix("==========step0:all check")
- # self.all_test()
+ self.all_test()
tdLog.printNoPrefix("==========step1:create table in normal database")
tdSql.prepare()
diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py
new file mode 100644
index 0000000000..49d6476d9c
--- /dev/null
+++ b/tests/system-test/1-insert/table_param_ttl.py
@@ -0,0 +1,79 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+
+class TDTestCase:
+ updatecfgDict = {'ttlUnit':5,'ttlPushInterval':3}
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+ self.ntbname = 'ntb'
+ self.stbname = 'stb'
+ self.tbnum = 10
+ self.ttl_param = 1
+ self.default_ttl = 100
+ self.modify_ttl = 1
+ def ttl_check_ntb(self):
+ tdSql.prepare()
+
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}')
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum)
+ sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(0)
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}')
+ for i in range(int(self.tbnum/2)):
+ tdSql.execute(f'alter table {self.ntbname}_{i} ttl {self.modify_ttl}')
+ sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum - int(self.tbnum/2))
+ tdSql.execute('drop database db')
+ def ttl_check_ctb(self):
+ tdSql.prepare()
+ tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)')
+
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param}')
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum)
+ sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(0)
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.default_ttl}')
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum)
+ for i in range(int(self.tbnum/2)):
+ tdSql.execute(f'alter table {self.stbname}_{i} ttl {self.modify_ttl}')
+ sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'])
+ tdSql.query(f'show tables')
+ tdSql.checkRows(self.tbnum - int(self.tbnum/2))
+ tdSql.execute('drop database db')
+
+ def run(self):
+ self.ttl_check_ntb()
+ self.ttl_check_ctb()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py
index 5387970d56..d4434987a6 100644
--- a/tests/system-test/1-insert/time_range_wise.py
+++ b/tests/system-test/1-insert/time_range_wise.py
@@ -17,25 +17,34 @@ TINT_COL = "c_tint"
FLOAT_COL = "c_float"
DOUBLE_COL = "c_double"
BOOL_COL = "c_bool"
-TINT_UN_COL = "c_tint_un"
-SINT_UN_COL = "c_sint_un"
-BINT_UN_COL = "c_bint_un"
-INT_UN_COL = "c_int_un"
+TINT_UN_COL = "c_utint"
+SINT_UN_COL = "c_usint"
+BINT_UN_COL = "c_ubint"
+INT_UN_COL = "c_uint"
BINARY_COL = "c_binary"
NCHAR_COL = "c_nchar"
TS_COL = "c_ts"
-
-
NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
CHAR_COL = [BINARY_COL, NCHAR_COL, ]
BOOLEAN_COL = [BOOL_COL, ]
TS_TYPE_COL = [TS_COL, ]
+INT_TAG = "t_int"
+
+ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL]
+TAG_COL = [INT_TAG]
+
# insert data args：
TIME_STEP = 10000
NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
+# init db/table
+DBNAME = "db"
+STBNAME = "stb1"
+CTBNAME = "ct1"
+NTBNAME = "nt1"
+
@dataclass
class DataSet:
@@ -73,19 +82,25 @@ class DataSet:
@dataclass
class SMAschema:
- creation : str = "CREATE"
- index_name : str = "sma_index_1"
- index_flag : str = "SMA INDEX"
- operator : str = "ON"
- tbname : str = None
- watermark : str = None
- maxdelay : str = None
- func : Tuple[str] = None
- interval : Tuple[str] = None
- sliding : str = None
- other : Any = None
- drop : str = "DROP"
- drop_flag : str = "INDEX"
+ creation : str = "CREATE"
+ index_name : str = "sma_index_1"
+ index_flag : str = "SMA INDEX"
+ operator : str = "ON"
+ tbname : str = None
+ watermark : str = "5s"
+ max_delay : str = "6m"
+ func : Tuple[str] = None
+ interval : Tuple[str] = ("6m", "10s")
+ sliding : str = "6m"
+ other : Any = None
+ drop : str = "DROP"
+ drop_flag : str = "INDEX"
+ querySmaOptimize : int = 1
+ show : str = "SHOW"
+ show_msg : str = "INDEXES"
+ show_oper : str = "FROM"
+ dbname : str = None
+ rollup_db : bool = False
def __post_init__(self):
if isinstance(self.other, dict):
@@ -111,8 +126,8 @@ class SMAschema:
self.watermark = v
del self.other[k]
- if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay:
- self.maxdelay = v
+ if k.lower() == "max_delay" and isinstance(v, str) and not self.max_delay:
+ self.max_delay = v
del self.other[k]
if k.lower() == "functions" and isinstance(v, tuple) and not self.func:
@@ -131,12 +146,36 @@ class SMAschema:
self.drop_flag = v
del self.other[k]
+ if k.lower() == "show_msg" and isinstance(v, str) and not self.show_msg:
+ self.show_msg = v
+ del self.other[k]
+
+ if k.lower() == "dbname" and isinstance(v, str) and not self.dbname:
+ self.dbname = v
+ del self.other[k]
+
+ if k.lower() == "show_oper" and isinstance(v, str) and not self.show_oper:
+ self.show_oper = v
+ del self.other[k]
+
+ if k.lower() == "rollup_db" and isinstance(v, bool) and not self.rollup_db:
+ self.rollup_db = v
+ del self.other[k]
+
+
+
+# from ...pytest.util.sql import *
+# from ...pytest.util.constant import *
class TDTestCase:
+ updatecfgDict = {"querySmaOptimize": 1}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
+ self.precision = "ms"
+ self.sma_count = 0
+ self.sma_created_index = []
"""
create sma index :
@@ -155,13 +194,17 @@ class TDTestCase:
if sma.func:
sql += f" function({', '.join(sma.func)})"
if sma.interval:
- sql += f" interval({', '.join(sma.interval)})"
+ interval, offset = self.__get_interval_offset(sma.interval)
+ if offset:
+ sql += f" interval({interval}, {offset})"
+ else:
+ sql += f" interval({interval})"
if sma.sliding:
sql += f" sliding({sma.sliding})"
if sma.watermark:
sql += f" watermark {sma.watermark}"
- if sma.maxdelay:
- sql += f" maxdelay {sma.maxdelay}"
+ if sma.max_delay:
+ sql += f" max_delay {sma.max_delay}"
if isinstance(sma.other, dict):
for k,v in sma.other.items():
if isinstance(v,tuple) or isinstance(v, list):
@@ -171,53 +214,305 @@ class TDTestCase:
if isinstance(sma.other, tuple) or isinstance(sma.other, list):
sql += " ".join(sma.other)
if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str):
- sql += sma.other
+ sql += f" {sma.other}"
return sql
- def sma_create_check(self, sma:SMAschema):
+ def __get_sma_func_col(self, func):
+ cols = []
+ if isinstance(func, str):
+ cols.append( func.split("(")[-1].split(")")[0] )
+ elif isinstance(func, tuple) or isinstance(func, list):
+ for func_col in func:
+ cols.append(func_col.split("(")[-1].split(")")[0])
+ else:
+ cols = []
+ return cols
+
+ def __check_sma_func(self, func:tuple):
+ if not isinstance(func, str) and not isinstance(func, tuple) and not isinstance(func, list):
+ return False
+ if isinstance(func, str) :
+ if "(" not in func or ")" not in func:
+ return False
+ if func.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS:
+ return False
+ if func.split("(")[1].split(")")[0] not in ALL_COL and func.split("(")[1].split(")")[0] not in TAG_COL :
+ return False
+ if isinstance(func, tuple) or isinstance(func, list):
+ for arg in func:
+ if not isinstance(arg, str):
+ return False
+ if "(" not in arg or ")" not in arg:
+ return False
+ if arg.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS:
+ return False
+ if arg.split("(")[1].split(")")[0] not in ALL_COL and arg.split("(")[1].split(")")[0] not in TAG_COL :
+ return False
+ return True
+
+ def __check_sma_watermark(self, arg):
+ if not arg:
+ return False
+ if not isinstance(arg, str):
+ return False
+ if arg[-1] not in SMA_WATMARK_MAXDELAY_INIT:
+ return False
+ if len(arg) == 1:
+ return False
+ if not arg[:-1].isdecimal():
+ return False
+ if tdSql.get_times(arg) > WATERMARK_MAX:
+ return False
+ if tdSql.get_times(arg) < WATERMARK_MIN:
+ return False
+
+ return True
+
+ def __check_sma_max_delay(self, arg):
+ if not self.__check_sma_watermark(arg):
+ return False
+ if tdSql.get_times(arg) < MAX_DELAY_MIN:
+ return False
+
+ return True
+
+ def __check_sma_sliding(self, arg):
+ if not isinstance(arg, str):
+ return False
+ if arg[-1] not in TAOS_TIME_INIT:
+ return False
+ if len(arg) == 1:
+ return False
+ if not arg[:-1].isdecimal():
+ return False
+
+ return True
+
+ def __get_interval_offset(self, args):
+ if isinstance(args, str):
+ interval, offset = args, None
+ elif isinstance(args,tuple) or isinstance(args, list):
+ if len(args) == 1:
+ interval, offset = args[0], None
+ elif len(args) == 2:
+ interval, offset = args
+ else:
+ interval, offset = False, False
+ else:
+ interval, offset = False, False
+
+ return interval, offset
+
+ def __check_sma_interval(self, args):
+ if not isinstance(args, tuple) and not isinstance(args,str):
+ return False
+ interval, offset = self.__get_interval_offset(args)
+ if not interval:
+ return False
+ if not self.__check_sma_sliding(interval):
+ return False
+ if tdSql.get_times(interval) < INTERVAL_MIN:
+ return False
+ if offset:
+ if not self.__check_sma_sliding(offset):
+ return False
+ if tdSql.get_times(interval) <= tdSql.get_times(offset) :
+ return False
+
+ return True
+
+ def __sma_create_check(self, sma:SMAschema):
+ if self.updatecfgDict["querySmaOptimize"] == 0:
+ return False
+ # # TODO: if database is a rollup-db, can not create sma index
+ # tdSql.query("select database()")
+ # if sma.rollup_db :
+ # return False
tdSql.query("show stables")
+ if not sma.tbname:
+ return False
stb_in_list = False
for row in tdSql.queryResult:
if sma.tbname == row[0]:
stb_in_list = True
- break
if not stb_in_list:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.creation:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.index_flag:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.index_name:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.operator:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.tbname:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.func:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.interval:
- tdSql.error(self.__create_sma_index(sma))
- if not sma.sliding:
- tdSql.error(self.__create_sma_index(sma))
+ return False
+ if not sma.creation or not isinstance(sma.creation, str) or sma.creation.upper() != "CREATE":
+ return False
+ if not sma.index_flag or not isinstance(sma.index_flag, str) or sma.index_flag.upper() != "SMA INDEX" :
+ return False
+ if not sma.index_name or not isinstance(sma.index_name, str) or sma.index_name.upper() in TAOS_KEYWORDS:
+ return False
+ if not sma.operator or not isinstance(sma.operator, str) or sma.operator.upper() != "ON":
+ return False
+
+ if not sma.func or not self.__check_sma_func(sma.func):
+ return False
+ tdSql.query(f"desc {sma.tbname}")
+ _col_list = []
+ for col_row in tdSql.queryResult:
+ _col_list.append(col_row[0])
+ _sma_func_cols = self.__get_sma_func_col(sma.func)
+ for _sma_func_col in _sma_func_cols:
+ if _sma_func_col not in _col_list:
+ return False
+
+ if sma.sliding and not self.__check_sma_sliding(sma.sliding):
+ return False
+ interval, _ = self.__get_interval_offset(sma.interval)
+ if not sma.interval or not self.__check_sma_interval(sma.interval) :
+ return False
+ if sma.sliding and tdSql.get_times(interval) < tdSql.get_times(sma.sliding):
+ return False
+ if sma.watermark and not self.__check_sma_watermark(sma.watermark):
+ return False
+ if sma.max_delay and not self.__check_sma_max_delay(sma.max_delay):
+ return False
if sma.other:
+ return False
+
+ return True
+
+ def sma_create_check(self, sma:SMAschema):
+ if self.__sma_create_check(sma):
+ tdSql.query(self.__create_sma_index(sma))
+ self.sma_count += 1
+ self.sma_created_index.append(sma.index_name)
+ tdSql.query("show streams")
+ tdSql.checkRows(self.sma_count)
+
+ else:
tdSql.error(self.__create_sma_index(sma))
+ def __drop_sma_index(self, sma:SMAschema):
+ sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}"
+ return sql
+
+ def __sma_drop_check(self, sma:SMAschema):
+ if not sma.drop:
+ return False
+ if not sma.drop_flag:
+ return False
+ if not sma.index_name:
+ return False
+
+ return True
+
+ def sma_drop_check(self, sma:SMAschema):
+ if self.__sma_drop_check(sma):
+ tdSql.query(self.__drop_sma_index(sma))
+ print(self.__drop_sma_index(sma))
+ self.sma_count -= 1
+ self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index))
+ tdSql.query("show streams")
+ tdSql.checkRows(self.sma_count)
+
+ else:
+ tdSql.error(self.__drop_sma_index(sma))
+
+ def __show_sma_index(self, sma:SMAschema):
+ sql = f"{sma.show} {sma.show_msg} {sma.show_oper} {sma.tbname}"
+ return sql
+
+ def __sma_show_check(self, sma:SMAschema):
+ if not sma.show:
+ return False
+ if not sma.show_msg:
+ return False
+ if not sma.show_oper:
+ return False
+ if not sma.tbname:
+ return False
+
+ return True
+
+ def sma_show_check(self, sma:SMAschema):
+ if self.__sma_show_check(sma):
+ tdSql.query(self.__show_sma_index(sma))
+ tdSql.checkRows(self.sma_count)
+ else:
+ tdSql.error(self.__show_sma_index(sma))
+
+ @property
+ def __create_sma_sql(self):
+ err_sqls = []
+ cur_sqls = []
+ # err_set
+ # # case 1: required fields check
+ err_sqls.append( SMAschema(creation="", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(index_name="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(index_flag="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(operator="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(tbname="", func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(func=("",),tbname=STBNAME ) )
+ err_sqls.append( SMAschema(interval=(""),tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+
+ # # case 2: err fields
+ err_sqls.append( SMAschema(creation="show",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(creation="alter",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(creation="select",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+
+ err_sqls.append( SMAschema(index_flag="SMA INDEXES", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+
+
+ # current_set
+
+ cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+ cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
+
+
+ return err_sqls, cur_sqls
+
+ def test_create_sma(self):
+ err_sqls , cur_sqls = self.__create_sma_sql
+ for err_sql in err_sqls:
+ self.sma_create_check(err_sql)
+ for cur_sql in cur_sqls:
+ self.sma_create_check(cur_sql)
+
+ @property
+ def __drop_sma_sql(self):
+ err_sqls = []
+ cur_sqls = []
+ # err_set
+ ## case 1: required fields check
+ err_sqls.append( SMAschema(drop="") )
+ err_sqls.append( SMAschema(drop_flag="") )
+ err_sqls.append( SMAschema(index_name="") )
+
+ for index in self.sma_created_index:
+ cur_sqls.append(SMAschema(index_name=index))
+
+ return err_sqls, cur_sqls
+
+ def test_drop_sma(self):
+ err_sqls , cur_sqls = self.__drop_sma_sql
+ for err_sql in err_sqls:
+ self.sma_drop_check(err_sql)
+ # for cur_sql in cur_sqls:
+ # self.sma_drop_check(cur_sql)
def all_test(self):
+ self.test_create_sma()
+ self.test_drop_sma()
+
pass
def __create_tb(self):
tdLog.printNoPrefix("==========step: create table")
- create_stb_sql = f'''create table stb1(
+ create_stb_sql = f'''create table {STBNAME}(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp,
{TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned,
{INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned
- ) tags (tag1 int)
+ ) tags ({INT_TAG} int)
'''
- create_ntb_sql = f'''create table t1(
+ create_ntb_sql = f'''create table {NTBNAME}(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp,
@@ -253,6 +548,7 @@ class TDTestCase:
return data_set
def __insert_data(self):
+ tdLog.printNoPrefix("==========step: start inser data into tables now.....")
data = self.__data_set(rows=self.rows)
# now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
@@ -278,7 +574,7 @@ class TDTestCase:
tdSql.execute(
f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )")
tdSql.execute(
- f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )")
+ f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )")
tdSql.execute(
f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )")
@@ -295,28 +591,31 @@ class TDTestCase:
f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )")
tdSql.execute(
- f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )")
+ f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )")
tdSql.execute(
- f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )")
+ f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )")
tdSql.execute(
- f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )")
+ f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )")
def run(self):
- sma1 = SMAschema(func=("min(c1)","max(c2)"))
- sql1 = self.__create_sma_index(sma1)
- print("================")
- print(sql1)
- # a = DataSet()
- # return
self.rows = 10
tdLog.printNoPrefix("==========step0:all check")
- # self.all_test()
tdLog.printNoPrefix("==========step1:create table in normal database")
tdSql.prepare()
self.__create_tb()
- self.__insert_data()
+ # self.__insert_data()
+ self.all_test()
+
+ # drop databases, create same name db、stb and sma index
+ # tdSql.prepare()
+ # self.__create_tb()
+ # self.__insert_data()
+ # self.all_test()
+
+
+
return
tdLog.printNoPrefix("==========step2:create table in rollup database")
diff --git a/tests/system-test/2-query/bottom.py b/tests/system-test/2-query/bottom.py
index 1037b0a8f3..a95daf22f4 100644
--- a/tests/system-test/2-query/bottom.py
+++ b/tests/system-test/2-query/bottom.py
@@ -17,100 +17,140 @@ from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
-
+from util.sqlset import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
-
+ self.dbname = 'db_test'
+ self.setsql = TDSetSql()
+ self.ntbname = 'ntb'
self.rowNum = 10
self.tbnum = 20
self.ts = 1537146000000
self.binary_str = 'taosdata'
self.nchar_str = '涛思数据'
- def bottom_check_base(self):
- tdSql.prepare()
- tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned,
- col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''')
- tdSql.execute("create table stb_1 using stb tags('beijing')")
- column_list = ['col1','col2','col3','col4','col5','col6','col7','col8']
- error_column_list = ['col11','col12','col13']
- error_param_list = [0,101]
- for i in range(self.rowNum):
- tdSql.execute(f"insert into stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')"
- % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
+ self.column_dict = {
+ 'ts' : 'timestamp',
+ 'col1': 'tinyint',
+ 'col2': 'smallint',
+ 'col3': 'int',
+ 'col4': 'bigint',
+ 'col5': 'tinyint unsigned',
+ 'col6': 'smallint unsigned',
+ 'col7': 'int unsigned',
+ 'col8': 'bigint unsigned',
+ 'col9': 'float',
+ 'col10': 'double',
+ 'col11': 'bool',
+ 'col12': 'binary(20)',
+ 'col13': 'nchar(20)'
+ }
- for i in column_list:
- tdSql.query(f'select bottom({i},2) from stb_1')
- tdSql.checkRows(2)
- tdSql.checkEqual(tdSql.queryResult,[(2,),(1,)])
- for j in error_param_list:
- tdSql.error(f'select bottom({i},{j}) from stb_1')
- for i in error_column_list:
- tdSql.error(f'select bottom({i},10) from stb_1')
- tdSql.query("select ts,bottom(col1, 2),ts from stb_1 group by tbname")
- tdSql.checkRows(2)
- tdSql.query('select bottom(col2,1) from stb_1 interval(1y) order by col2')
- tdSql.checkData(0,0,1)
-
- tdSql.error('select * from stb_1 where bottom(col2,1)=1')
- tdSql.execute('drop database db')
- def bottom_check_distribute(self):
- # prepare data for vgroup 4
- dbname = tdCom.getLongName(5, "letters")
+ self.param_list = [1,100]
+ def insert_data(self,column_dict,tbname,row_num):
+ sql = ''
+ for k, v in column_dict.items():
+ if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \
+ v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool':
+ sql += '%d,'
+ elif v.lower() == 'float' or v.lower() == 'double':
+ sql += '%f,'
+ elif 'binary' in v.lower():
+ sql += f'"{self.binary_str}%d",'
+ elif 'nchar' in v.lower():
+ sql += f'"{self.nchar_str}%d",'
+ insert_sql = f'insert into {tbname} values({sql[:-1]})'
+ for i in range(row_num):
+ insert_list = []
+ for k, v in column_dict.items():
+ if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\
+ 'binary' in v.lower() or 'nchar' in v.lower():
+ insert_list.append(0 + i)
+ elif v.lower() == 'float' or v.lower() == 'double':
+ insert_list.append(0.1 + i)
+ elif v.lower() == 'bool':
+ insert_list.append(i % 2)
+ elif v.lower() == 'timestamp':
+ insert_list.append(self.ts + i)
+ tdSql.execute(insert_sql%(tuple(insert_list)))
+ def bottom_check_data(self,tbname,tb_type):
+ new_column_dict = {}
+ for param in self.param_list:
+ for k,v in self.column_dict.items():
+ if v.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']:
+ tdSql.query(f'select bottom({k},{param}) from {tbname} order by {k}')
+ if param >= self.rowNum:
+ if tb_type in ['normal_table','child_table']:
+ tdSql.checkRows(self.rowNum)
+ values_list = []
+ for i in range(self.rowNum):
+ tp = (i,)
+ values_list.append(tp)
+ tdSql.checkEqual(tdSql.queryResult,values_list)
+ elif tb_type == 'stable':
+ tdSql.checkRows(param)
+ elif param < self.rowNum:
+ if tb_type in ['normal_table','child_table']:
+ tdSql.checkRows(param)
+ values_list = []
+ for i in range(param):
+ tp = (i,)
+ values_list.append(tp)
+ tdSql.checkEqual(tdSql.queryResult,values_list)
+ elif tb_type == 'stable':
+ tdSql.checkRows(param)
+ for i in [self.param_list[0]-1,self.param_list[-1]+1]:
+ tdSql.error(f'select top({k},{i}) from {tbname}')
+ new_column_dict.update({k:v})
+ elif v.lower() == 'bool' or 'binary' in v.lower() or 'nchar' in v.lower():
+ tdSql.error(f'select top({k},{param}) from {tbname}')
+ tdSql.error(f'select * from {tbname} where top({k},{param})=1')
+ pass
+ def bottom_check_ntb(self):
+ tdSql.execute(f'create database if not exists {self.dbname} vgroups 1')
+ tdSql.execute(f'use {self.dbname}')
+ tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
+ self.insert_data(self.column_dict,self.ntbname,self.rowNum)
+ self.bottom_check_data(self.ntbname,'normal_table')
+ tdSql.execute(f'drop database {self.dbname}')
+ def bottom_check_stb(self):
stbname = tdCom.getLongName(5, "letters")
- vgroup_num = 2
- child_table_num = 20
- tdSql.execute(f"create database if not exists {dbname} vgroups {vgroup_num}")
- tdSql.execute(f'use {dbname}')
- # build 20 child tables,every table insert 10 rows
- tdSql.execute(f'''create table {stbname}(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned,
- col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''')
- for i in range(child_table_num):
- tdSql.execute(f"create table {stbname}_{i} using {stbname} tags('beijing')")
- tdSql.execute(f"insert into {stbname}_{i}(ts) values(%d)" % (self.ts - 1-i))
- column_list = ['col1','col2','col3','col4','col5','col6','col7','col8']
- error_column_list = ['col11','col12','col13']
- error_param_list = [0,101]
- for i in [f'{stbname}', f'{dbname}.{stbname}']:
- for j in column_list:
- tdSql.query(f"select bottom({j},1) from {i}")
- tdSql.checkRows(0)
+ tag_dict = {
+ 't0':'int'
+ }
+ tag_values = [
+ f'1'
+ ]
+ tdSql.execute(f"create database if not exists {self.dbname} vgroups 2")
+ tdSql.execute(f'use {self.dbname}')
+ tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict))
+ for i in range(self.tbnum):
+ tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})")
+ tdSql.execute(self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum))
tdSql.query('show tables')
vgroup_list = []
for i in range(len(tdSql.queryResult)):
vgroup_list.append(tdSql.queryResult[i][6])
vgroup_list_set = set(vgroup_list)
-
for i in vgroup_list_set:
vgroups_num = vgroup_list.count(i)
- if vgroups_num >=2:
+ if vgroups_num >= 2:
tdLog.info(f'This scene with {vgroups_num} vgroups is ok!')
- continue
else:
- tdLog.exit(f'This scene does not meet the requirements with {vgroups_num} vgroup!\n')
- for i in range(self.rowNum):
- for j in range(child_table_num):
- tdSql.execute(f"insert into {stbname}_{j} values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')"
- % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
- for i in column_list:
- tdSql.query(f'select bottom({i},2) from {stbname}')
- tdSql.checkRows(2)
- tdSql.checkEqual(tdSql.queryResult,[(1,),(1,)])
- for j in error_param_list:
- tdSql.error(f'select bottom({i},{j}) from {stbname}')
- for i in error_column_list:
- tdSql.error(f'select bottom({i},10) from {stbname}')
-
- tdSql.execute(f'drop database {dbname}')
+ tdLog.exit(
+ 'This scene does not meet the requirements with {vgroups_num} vgroup!\n')
+ for i in range(self.tbnum):
+ self.bottom_check_data(f'{stbname}_{i}','child_table')
+ self.bottom_check_data(f'{stbname}','stable')
+ tdSql.execute(f'drop database {self.dbname}')
+
def run(self):
-
- self.bottom_check_base()
- self.bottom_check_distribute()
+ self.bottom_check_ntb()
+ self.bottom_check_stb()
-
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py
index 387170991f..934bbbd7b4 100644
--- a/tests/system-test/2-query/cast.py
+++ b/tests/system-test/2-query/cast.py
@@ -630,27 +630,27 @@ class TDTestCase:
( tdSql.checkData(i, 0, '12') for i in range(tdSql.queryRows) )
tdLog.printNoPrefix("==========step40: error cast condition, should return error ")
- tdSql.error("select cast(c1 as int) as b from ct4")
- tdSql.error("select cast(c1 as bool) as b from ct4")
- tdSql.error("select cast(c1 as tinyint) as b from ct4")
- tdSql.error("select cast(c1 as smallint) as b from ct4")
- tdSql.error("select cast(c1 as float) as b from ct4")
- tdSql.error("select cast(c1 as double) as b from ct4")
- tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4")
- tdSql.error("select cast(c1 as smallint unsigned) as b from ct4")
- tdSql.error("select cast(c1 as int unsigned) as b from ct4")
+ #tdSql.error("select cast(c1 as int) as b from ct4")
+ #tdSql.error("select cast(c1 as bool) as b from ct4")
+ #tdSql.error("select cast(c1 as tinyint) as b from ct4")
+ #tdSql.error("select cast(c1 as smallint) as b from ct4")
+ #tdSql.error("select cast(c1 as float) as b from ct4")
+ #tdSql.error("select cast(c1 as double) as b from ct4")
+ #tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4")
+ #tdSql.error("select cast(c1 as smallint unsigned) as b from ct4")
+ #tdSql.error("select cast(c1 as int unsigned) as b from ct4")
- tdSql.error("select cast(c2 as int) as b from ct4")
- tdSql.error("select cast(c3 as bool) as b from ct4")
- tdSql.error("select cast(c4 as tinyint) as b from ct4")
- tdSql.error("select cast(c5 as smallint) as b from ct4")
- tdSql.error("select cast(c6 as float) as b from ct4")
- tdSql.error("select cast(c7 as double) as b from ct4")
- tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4")
+ #tdSql.error("select cast(c2 as int) as b from ct4")
+ #tdSql.error("select cast(c3 as bool) as b from ct4")
+ #tdSql.error("select cast(c4 as tinyint) as b from ct4")
+ #tdSql.error("select cast(c5 as smallint) as b from ct4")
+ #tdSql.error("select cast(c6 as float) as b from ct4")
+ #tdSql.error("select cast(c7 as double) as b from ct4")
+ #tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4")
- tdSql.error("select cast(c8 as timestamp ) as b from ct4")
- tdSql.error("select cast(c9 as timestamp ) as b from ct4")
- tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
+ #tdSql.error("select cast(c8 as timestamp ) as b from ct4")
+ #tdSql.error("select cast(c9 as timestamp ) as b from ct4")
+ #tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
pass
def run(self):
diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py
index 5ff11c84dd..df6390f59c 100644
--- a/tests/system-test/2-query/join.py
+++ b/tests/system-test/2-query/join.py
@@ -28,7 +28,7 @@ class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
- tdSql.init(conn.cursor(), True)
+ tdSql.init(conn.cursor(), False)
def __query_condition(self,tbname):
query_condition = []
diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py
index 40da41eee7..5533cb840e 100644
--- a/tests/system-test/2-query/join2.py
+++ b/tests/system-test/2-query/join2.py
@@ -28,7 +28,7 @@ class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
- tdSql.init(conn.cursor(), True)
+ tdSql.init(conn.cursor(), False)
def __query_condition(self,tbname):
query_condition = []
diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py
index 26d4d63958..6c36ffc192 100644
--- a/tests/system-test/2-query/json_tag.py
+++ b/tests/system-test/2-query/json_tag.py
@@ -11,12 +11,14 @@
# -*- coding: utf-8 -*-
+import imp
import sys
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
import json
+import os
class TDTestCase:
@@ -29,6 +31,9 @@ class TDTestCase:
return
def init(self, conn, logSql):
+ self.testcasePath = os.path.split(__file__)[0]
+ self.testcaseFilename = os.path.split(__file__)[-1]
+ os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
@@ -568,6 +573,103 @@ class TDTestCase:
tdSql.checkRows(3)
tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1")
tdSql.checkRows(3)
+
+ #math function
+ tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select cos(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select tan(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select asin(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select acos(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select atan(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select abs(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select pow(dataint,5) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select log(dataint,10) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select sqrt(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select HISTOGRAM(dataint,'user_input','[1, 33, 555, 7777]',1) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select csum(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select mavg(dataint,1) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select statecount(dataint,'GE',10) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select stateduration(dataint,'GE',0) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select sample(dataint,3) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select HYPERLOGLOG(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(1)
+ tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(1)
+
+ # function not ready
+ # tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(3)
+ # tdSql.query("select unique(dataint) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(3)
+ # tdSql.query("select mode(dataint) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(3)
+ # tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;")
+ # tdSql.checkRows(1)
+
+ #str function
+ tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select ltrim(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select lower(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select rtrim(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CHAR_LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select SUBSTR(dataStr,5) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CONCAT(dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CONCAT_ws('adad!@!@%$^$%$^$%^a',dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select CAST(dataStr as bigint) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+
+ #time function
+ tdSql.query("select now() from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select today() from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMEZONE() from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TO_ISO8601(ts) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMETRUNCATE(ts,1u) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select TIMEDIFF(ts,1u) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(3)
+ tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;")
+ tdSql.checkRows(1)
+
#
# #test TD-12077
tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')")
diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py
index 79b5ac515b..f833a42b57 100644
--- a/tests/system-test/2-query/substr.py
+++ b/tests/system-test/2-query/substr.py
@@ -31,7 +31,7 @@ class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
- tdSql.init(conn.cursor())
+ tdSql.init(conn.cursor(),False)
def __substr_condition(self): # sourcery skip: extract-method
substr_condition = []
diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py
index 0e2110f912..a4c85cc5ed 100644
--- a/tests/system-test/2-query/tail.py
+++ b/tests/system-test/2-query/tail.py
@@ -188,8 +188,8 @@ class TDTestCase:
def check_tail_table(self , tbname , col_name , tail_rows , offset):
tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}"
- equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts"
- #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}"
+ #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts"
+ equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}"
tdSql.query(tail_sql)
tail_result = tdSql.queryResult
@@ -404,7 +404,7 @@ class TDTestCase:
f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
- tdSql.query("select tail(c2,2) from sub1_bound")
+ tdSql.query("select tail(c2,2) from sub1_bound order by 1 desc")
tdSql.checkRows(2)
tdSql.checkData(0,0,9223372036854775803)
diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py
index 75134224db..5f4ab7357b 100644
--- a/tests/system-test/6-cluster/5dnode1mnode.py
+++ b/tests/system-test/6-cluster/5dnode1mnode.py
@@ -20,7 +20,7 @@ class MyDnodes(TDDnodes):
self.simDeployed = False
class TDTestCase:
-
+ noConn = True
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
self.TDDnodes = None
@@ -40,7 +40,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
- if ("taosd" in files):
+ if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
@@ -81,7 +81,7 @@ class TDTestCase:
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
- cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+ cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
print(cmd)
os.system(cmd)
diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py
index d3cde987c6..e08e738be6 100644
--- a/tests/system-test/6-cluster/5dnode2mnode.py
+++ b/tests/system-test/6-cluster/5dnode2mnode.py
@@ -20,6 +20,7 @@ class MyDnodes(TDDnodes):
self.simDeployed = False
class TDTestCase:
+ noConn = True
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
@@ -40,7 +41,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
- if ("taosd" in files):
+ if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
@@ -85,7 +86,7 @@ class TDTestCase:
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
- cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+ cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
print(cmd)
os.system(cmd)
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py
new file mode 100644
index 0000000000..59fe1c0b16
--- /dev/null
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py
@@ -0,0 +1,181 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+ def init(self,conn ,logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ self.TDDnodes = None
+ tdSql.init(conn.cursor())
+ self.host = socket.gethostname()
+
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def _async_raise(self, tid, exctype):
+ """raises the exception, performs cleanup if needed"""
+ if not inspect.isclass(exctype):
+ exctype = type(exctype)
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+ if res == 0:
+ raise ValueError("invalid thread id")
+ elif res != 1:
+ # """if it returns a number greater than one, you're in trouble,
+ # and you should call it again with exc=NULL to revert the effect"""
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+ raise SystemError("PyThreadState_SetAsyncExc failed")
+
+ def stopThread(self,thread):
+ self._async_raise(thread.ident, SystemExit)
+
+
+ def insertData(self,countstart,countstop):
+ # fisrt add data : db\stable\childtable\general table
+
+ for couti in range(countstart,countstop):
+ tdLog.debug("drop database if exists db%d" %couti)
+ tdSql.execute("drop database if exists db%d" %couti)
+ print("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("use db%d" %couti)
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ dbNumbers = int(dnodenumbers * restartNumber)
+
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
+
+ # fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
+ tdSql.execute("create mnode on dnode 2")
+ clusterComCheck.checkMnodeStatus(2)
+ tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
+
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
+ tdSql.error("create mnode on dnode 2")
+ tdSql.query("show dnodes;")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+
+ tdLog.info("Take turns stopping all dnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
+ tdDnodes=cluster.dnodes
+ stopcount =0
+ while stopcount < restartNumber:
+ for i in range(dnodenumbers):
+ # threads=[]
+ # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
+ paraDict["dbName"]= 'db%d%d'%(stopcount,i)
+ threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))
+ threads.start()
+ tdDnodes[i].stoptaosd()
+ # sleep(10)
+ tdDnodes[i].starttaosd()
+ # sleep(10)
+
+ if clusterComCheck.checkDnodes(dnodenumbers):
+ # threads.join()
+ tdLog.info("first restart loop")
+ else:
+ print("456")
+ threads.join()
+ self.stopThread(threads)
+ tdLog.exit("one or more of dnodes failed to start ")
+ # self.check3mnode()
+ stopcount+=1
+ threads.join()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkDbRows(dbNumbers)
+ for i in range(restartNumber):
+ clusterComCheck.checkDb(dnodenumbers,'db%d'%i)
+
+
+ def run(self):
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py
new file mode 100644
index 0000000000..cf608f6480
--- /dev/null
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py
@@ -0,0 +1,181 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+ def init(self,conn ,logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ self.TDDnodes = None
+ tdSql.init(conn.cursor())
+ self.host = socket.gethostname()
+
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def _async_raise(self, tid, exctype):
+ """raises the exception, performs cleanup if needed"""
+ if not inspect.isclass(exctype):
+ exctype = type(exctype)
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+ if res == 0:
+ raise ValueError("invalid thread id")
+ elif res != 1:
+ # """if it returns a number greater than one, you're in trouble,
+ # and you should call it again with exc=NULL to revert the effect"""
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+ raise SystemError("PyThreadState_SetAsyncExc failed")
+
+ def stopThread(self,thread):
+ self._async_raise(thread.ident, SystemExit)
+
+
+ def insertData(self,countstart,countstop):
+ # fisrt add data : db\stable\childtable\general table
+
+ for couti in range(countstart,countstop):
+ tdLog.debug("drop database if exists db%d" %couti)
+ tdSql.execute("drop database if exists db%d" %couti)
+ print("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("use db%d" %couti)
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ dbNumbers = int(mnodeNums * restartNumber)
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
+
+ # fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
+ tdSql.execute("create mnode on dnode 2")
+ clusterComCheck.checkMnodeStatus(2)
+ tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
+
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
+ tdSql.error("create mnode on dnode 2")
+ tdSql.query("show dnodes;")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+
+ tdLog.info("Take turns stopping Mnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
+ tdDnodes=cluster.dnodes
+ stopcount =0
+ while stopcount < restartNumber:
+ tdLog.info("first restart loop")
+ for i in range(mnodeNums):
+ # threads=[]
+ # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
+ paraDict["dbName"]= 'db%d%d'%(stopcount,i)
+ threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))
+ threads.start()
+ tdDnodes[i].stoptaosd()
+ # sleep(10)
+ tdDnodes[i].starttaosd()
+ # sleep(10)
+
+ if clusterComCheck.checkDnodes(dnodenumbers):
+ # threads.join()
+ tdLog.info("123")
+ else:
+ print("456")
+ threads.join()
+ self.stopThread(threads)
+ tdLog.exit("one or more of dnodes failed to start ")
+ # self.check3mnode()
+ stopcount+=1
+ threads.join()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkDbRows(dbNumbers)
+ for i in range(restartNumber):
+ clusterComCheck.checkDb(mnodeNums,'db%d'%i)
+
+
+ def run(self):
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py
new file mode 100644
index 0000000000..2d2322fada
--- /dev/null
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py
@@ -0,0 +1,181 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+ def init(self,conn ,logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ self.TDDnodes = None
+ tdSql.init(conn.cursor())
+ self.host = socket.gethostname()
+
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def _async_raise(self, tid, exctype):
+ """raises the exception, performs cleanup if needed"""
+ if not inspect.isclass(exctype):
+ exctype = type(exctype)
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+ if res == 0:
+ raise ValueError("invalid thread id")
+ elif res != 1:
+ # """if it returns a number greater than one, you're in trouble,
+ # and you should call it again with exc=NULL to revert the effect"""
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+ raise SystemError("PyThreadState_SetAsyncExc failed")
+
+ def stopThread(self,thread):
+ self._async_raise(thread.ident, SystemExit)
+
+
+ def insertData(self,countstart,countstop):
+ # fisrt add data : db\stable\childtable\general table
+
+ for couti in range(countstart,countstop):
+ tdLog.debug("drop database if exists db%d" %couti)
+ tdSql.execute("drop database if exists db%d" %couti)
+ print("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+ tdSql.execute("use db%d" %couti)
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ vnodeNumbers = int(dnodenumbers-mnodeNums)
+ dbNumbers = int(vnodeNumbers * restartNumber)
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
+
+ # fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
+ tdSql.execute("create mnode on dnode 2")
+ clusterComCheck.checkMnodeStatus(2)
+ tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
+
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
+ tdSql.error("create mnode on dnode 2")
+ tdSql.query("show dnodes;")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+
+ tdLog.info("Take turns stopping Vnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
+ tdDnodes=cluster.dnodes
+ stopcount =0
+ while stopcount < restartNumber:
+ for i in range(vnodeNumbers):
+ # threads=[]
+ # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
+ paraDict["dbName"]= 'db%d%d'%(stopcount,i)
+ threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))
+ threads.start()
+ tdDnodes[mnodeNums+i].stoptaosd()
+ # sleep(10)
+ tdDnodes[mnodeNums+i].starttaosd()
+ # sleep(10)
+
+ if clusterComCheck.checkDnodes(vnodeNumbers):
+ # threads.join()
+ tdLog.info("first restart loop")
+ else:
+ print("456")
+ threads.join()
+ self.stopThread(threads)
+ tdLog.exit("one or more of dnodes failed to start ")
+ # self.check3mnode()
+ stopcount+=1
+ threads.join()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkDbRows(dbNumbers)
+ for i in range(restartNumber):
+ clusterComCheck.checkDb(vnodeNumbers,'db%d'%i)
+
+
+ def run(self):
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py
index 1739db09af..aa1d7ecc29 100644
--- a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py
+++ b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py
@@ -9,6 +9,11 @@ from util.sql import *
from util.cases import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
+from util.cluster import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
import time
import socket
import subprocess
@@ -17,26 +22,13 @@ import threading
import time
import inspect
import ctypes
-class MyDnodes(TDDnodes):
- def __init__(self ,dnodes_lists):
- super(MyDnodes,self).__init__()
- self.dnodes = dnodes_lists # dnode must be TDDnode instance
- self.simDeployed = False
-
class TDTestCase:
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
- self.TDDnodes = None
-
- def buildcluster(self,dnodenumber):
- self.depoly_cluster(dnodenumber)
- self.master_dnode = self.TDDnodes.dnodes[0]
- self.host=self.master_dnode.cfgDict["fqdn"]
- conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
- tdSql.init(conn1.cursor())
-
+ # tdSql.init(conn.cursor())
+ # self.host = socket.gethostname()
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -106,52 +98,6 @@ class TDTestCase:
tdSql.checkData(0,0,rowsPerSTable)
return
- def depoly_cluster(self ,dnodes_nums=5,independent=True):
-
- testCluster = False
- valgrind = 0
- hostname = socket.gethostname()
- dnodes = []
- start_port = 6030
- start_port_sec = 6130
- for num in range(1, dnodes_nums+1):
- dnode = TDDnode(num)
- dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
- dnode.addExtraCfg("fqdn", f"{hostname}")
- dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
- dnode.addExtraCfg("monitorFqdn", hostname)
- dnode.addExtraCfg("monitorPort", 7043)
- dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
- # configure three dnoe don't support vnodes
- if independent and (num < 4):
- dnode.addExtraCfg("supportVnodes", 0)
-
- dnodes.append(dnode)
-
- self.TDDnodes = MyDnodes(dnodes)
- self.TDDnodes.init("")
- self.TDDnodes.setTestCluster(testCluster)
- self.TDDnodes.setValgrind(valgrind)
- self.TDDnodes.stopAll()
- for dnode in self.TDDnodes.dnodes:
- self.TDDnodes.deploy(dnode.index,{})
-
- for dnode in self.TDDnodes.dnodes:
- self.TDDnodes.starttaosd(dnode.index)
-
- # create cluster
- for dnode in self.TDDnodes.dnodes[1:]:
- # print(dnode.cfgDict)
- dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
- dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
- dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
- cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
- print(cmd)
- os.system(cmd)
-
- time.sleep(2)
- tdLog.info(" create cluster with %d dnode done! " %dnodes_nums)
-
def checkdnodes(self,dnodenumber):
count=0
while count < 100:
@@ -305,6 +251,14 @@ class TDTestCase:
tdSql.checkData(2,2,'offline')
tdSql.checkData(2,3,'ready')
+
+ def check5dnode(self):
+ tdSql.query("show dnodes;")
+ tdSql.checkData(0,1,'%s:6030'%self.host)
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ tdSql.checkData(0,4,'ready')
+ tdSql.checkData(4,4,'ready')
+
def five_dnode_three_mnode(self,dnodenumber):
tdSql.query("show dnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
@@ -346,6 +300,7 @@ class TDTestCase:
threads.join()
else:
print("456")
+ threads.join()
self.stop_thread(threads)
assert 1 == 2 ,"some dnode started failed"
return False
@@ -357,16 +312,8 @@ class TDTestCase:
self.check3mnode()
- def getConnection(self, dnode):
- host = dnode.cfgDict["fqdn"]
- port = dnode.cfgDict["serverPort"]
- config_dir = dnode.cfgDir
- return taos.connect(host=host, port=int(port), config=config_dir)
-
-
def run(self):
# print(self.master_dnode.cfgDict)
- self.buildcluster(5)
self.five_dnode_three_mnode(5)
def stop(self):
diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py
index 3a85207af6..5311d29846 100644
--- a/tests/system-test/6-cluster/5dnode3mnodeStop.py
+++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py
@@ -12,7 +12,10 @@ from util.dnodes import TDDnodes
from util.dnodes import TDDnode
from util.cluster import *
from test import tdDnodes
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import *
import time
import socket
import subprocess
@@ -216,64 +219,88 @@ class TDTestCase:
else:
tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%dnodeNumbers)
- def five_dnode_three_mnode(self,dnodenumber):
- self.check_dnodes_status(5)
- tdSql.query("show mnodes;")
- tdLog.debug(self.host)
- tdSql.checkRows(1)
+ def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'replica': 1,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+ dnodenumbers=int(dnodenumbers)
+ mnodeNums=int(mnodeNums)
+ dbNumbers = int(dnodenumbers * restartNumber)
+
+ tdLog.info("first check dnode and mnode")
+ tdSql.query("show dnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
- tdSql.checkData(0,2,'leader')
- tdSql.checkData(0,3,'ready')
+ tdSql.checkData(4,1,'%s:6430'%self.host)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(1)
# fisr add three mnodes;
+ tdLog.info("fisr add three mnodes and check mnode status")
tdSql.execute("create mnode on dnode 2")
- time.sleep(10)
+ clusterComCheck.checkMnodeStatus(2)
tdSql.execute("create mnode on dnode 3")
+ clusterComCheck.checkMnodeStatus(3)
- # fisrt check statut ready
- self.check3mnode()
-
-
+ # add some error operations and
+ tdLog.info("Confirm the status of the dnode again")
tdSql.error("create mnode on dnode 2")
-
tdSql.query("show dnodes;")
- # tdLog.debug(tdSql.queryResult)
-
- tdLog.debug("stop and follower of mnode")
+ print(tdSql.queryResult)
+ clusterComCheck.checkDnodes(dnodenumbers)
+ # restart all taosd
tdDnodes=cluster.dnodes
- # tdLog.debug(tdDnodes[0])
tdDnodes[1].stoptaosd()
- self.check3mnode2off()
+ clusterComCheck.check3mnodeoff(2,3)
tdDnodes[1].starttaosd()
- self.check3mnode()
+ clusterComCheck.checkMnodeStatus(3)
tdDnodes[2].stoptaosd()
- self.check3mnode3off()
+ clusterComCheck.check3mnodeoff(3,3)
tdDnodes[2].starttaosd()
- self.check3mnode()
+ clusterComCheck.checkMnodeStatus(3)
tdDnodes[0].stoptaosd()
- self.check3mnode1off()
+ clusterComCheck.check3mnodeoff(1,3)
tdDnodes[0].starttaosd()
- self.check3mnode()
+ clusterComCheck.checkMnodeStatus(3)
- self.check3mnode()
+ tdLog.info("Take turns stopping all dnodes ")
+ # seperate vnode and mnode in different dnodes.
+ # create database and stable
stopcount =0
while stopcount <= 2:
- for i in range(dnodenumber):
+ tdLog.info("first restart loop")
+ for i in range(dnodenumbers):
tdDnodes[i].stoptaosd()
tdDnodes[i].starttaosd()
- # self.check3mnode()
stopcount+=1
- self.check3mnode()
+ clusterComCheck.checkDnodes(dnodenumbers)
+ clusterComCheck.checkMnodeStatus(3)
def run(self):
- self.five_dnode_three_mnode(5)
+ # print(self.master_dnode.cfgDict)
+ self.fiveDnodeThreeMnode(5,3,1)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
-tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py
new file mode 100644
index 0000000000..d030008963
--- /dev/null
+++ b/tests/system-test/6-cluster/clusterCommonCheck.py
@@ -0,0 +1,211 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from collections import defaultdict
+import random
+import string
+import threading
+import requests
+import time
+# import socketfrom
+
+import taos
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+# class actionType(Enum):
+# CREATE_DATABASE = 0
+# CREATE_STABLE = 1
+# CREATE_CTABLE = 2
+# INSERT_DATA = 3
+
+class ClusterComCheck:
+ def init(self, conn, logSql):
+ tdSql.init(conn.cursor())
+ # tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def checkDnodes(self,dnodeNumbers):
+ count=0
+ while count < 5:
+ tdSql.query("show dnodes")
+ # tdLog.debug(tdSql.queryResult)
+ status=0
+ for i in range(dnodeNumbers):
+ if tdSql.queryResult[i][4] == "ready":
+ status+=1
+ tdLog.info(status)
+
+ if status == dnodeNumbers:
+ tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within 5s! " %dnodeNumbers)
+ return True
+ count+=1
+ time.sleep(1)
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within 5s ! "%dnodeNumbers)
+
+ def checkDbRows(self,dbNumbers):
+ dbNumbers=int(dbNumbers)
+ count=0
+ while count < 5:
+ tdSql.query("show databases;")
+ if tdSql.checkRows(dbNumbers+2):
+ tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2))
+ return True
+ else:
+ continue
+ else :
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("we find %d databases but expect %d in clusters! " %(tdSql.queryRows,dbNumbers))
+
+ def checkDb(self,dbNumbers,dbindex):
+ count=0
+ while count < 5:
+ query_status=0
+ for i in range(dbNumbers):
+ for j in range(dbNumbers):
+ tdSql.query("show databases;")
+ if "%s%d"%(dbindex,j) == tdSql.queryResult[i+2][0] :
+ if tdSql.queryResult[i+2][19] == "ready":
+ query_status+=1
+ else:
+ continue
+ # print(query_status)
+ count+=1
+ if query_status == dbNumbers:
+ tdLog.success("we find cluster with %d dnode and check all databases are ready within 5s! " %dbNumbers)
+ return True
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("database is not ready within 5s")
+
+ def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,):
+ tdSql.execute("use %s"%dbname)
+ tdSql.query("show stables")
+ tdSql.checkRows(stableCount)
+ tdSql.query("show tables")
+ tdSql.checkRows(CtableCount)
+ for i in range(stableCount):
+ tdSql.query("select count(*) from %s%d"%(stbname,i))
+ tdSql.checkData(0,0,rowsPerSTable)
+ return
+
+ def checkMnodeStatus(self,mnodeNums):
+ self.mnodeNums=int(mnodeNums)
+ # self.leaderDnode=int(leaderDnode)
+
+ count=0
+
+ while count < 10:
+ time.sleep(1)
+ tdSql.query("show mnodes;")
+ if tdSql.checkRows(self.mnodeNums) :
+ tdLog.success("cluster has %d mnodes" %self.mnodeNums )
+
+ if self.mnodeNums == 1:
+ if tdSql.queryResult[0][2]== 'leader' and tdSql.queryResult[0][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ count+=1
+ elif self.mnodeNums == 3 :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ elif tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ elif tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' :
+ if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ count+=1
+ elif self.mnodeNums == 2 :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ elif tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums)
+ return True
+ count+=1
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("cluster of %d mnodes is not ready in 10s " %self.mnodeNums)
+
+
+
+
+ def check3mnodeoff(self,offlineDnodeNo,mnodeNums=3):
+ count=0
+ while count < 10:
+ time.sleep(1)
+ tdSql.query("show mnodes;")
+ if tdSql.checkRows(mnodeNums) :
+ tdLog.success("cluster has %d mnodes" %self.mnodeNums )
+ else:
+ tdLog.exit("mnode number is correct")
+ if offlineDnodeNo == 1:
+ if tdSql.queryResult[0][2]=='offline' :
+ if tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.success("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ elif tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ count+=1
+ elif offlineDnodeNo == 2:
+ if tdSql.queryResult[1][2]=='offline' :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ elif tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ count+=1
+ elif offlineDnodeNo == 3:
+ if tdSql.queryResult[2][2]=='offline' :
+ if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ elif tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' :
+ if tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' :
+ tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo)
+ return True
+ count+=1
+ else:
+ tdLog.debug(tdSql.queryResult)
+ tdLog.exit("stop mnodes on dnode %d failed in 10s ")
+
+
+
+
+
+
+ def close(self):
+ self.cursor.close()
+
+clusterComCheck = ClusterComCheck()
diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py
new file mode 100644
index 0000000000..b3107d8537
--- /dev/null
+++ b/tests/system-test/6-cluster/clusterCommonCreate.py
@@ -0,0 +1,298 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from collections import defaultdict
+import random
+import string
+import threading
+import requests
+import time
+# import socketfrom
+
+import taos
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+# class actionType(Enum):
+# CREATE_DATABASE = 0
+# CREATE_STABLE = 1
+# CREATE_CTABLE = 2
+# INSERT_DATA = 3
+
+class ClusterComCreate:
+ def init(self, conn, logSql):
+ tdSql.init(conn.cursor())
+ # tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def initConsumerTable(self,cdbName='cdb'):
+ tdLog.info("create consume database, and consume info table, and consume result table")
+ tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
+ tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
+ tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
+ tdSql.query("drop table if exists %s.notifyinfo "%(cdbName))
+
+ tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
+ tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
+ tdSql.query("create table %s.notifyinfo (ts timestamp, cmdid int, consumerid int)"%cdbName)
+
+ def initConsumerInfoTable(self,cdbName='cdb'):
+ tdLog.info("drop consumeinfo table")
+ tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
+ tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
+
+ def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'):
+ sql = "insert into %s.consumeinfo values "%cdbName
+ sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
+ tdLog.info("consume info sql: %s"%sql)
+ tdSql.query(sql)
+
+ def selectConsumeResult(self,expectRows,cdbName='cdb'):
+ resultList=[]
+ while 1:
+ tdSql.query("select * from %s.consumeresult"%cdbName)
+ #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+ if tdSql.getRows() == expectRows:
+ break
+ else:
+ time.sleep(5)
+
+ for i in range(expectRows):
+ tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3)))
+ resultList.append(tdSql.getData(i , 3))
+
+ return resultList
+
+ def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
+ buildPath = tdCom.getBuildPath()
+ cfgPath = tdCom.getClientCfgPath()
+ if valgrind == 1:
+ logFile = cfgPath + '/../log/valgrind-tmq.log'
+ shellCmd = 'nohup valgrind --log-file=' + logFile
+ shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
+
+ if (platform.system().lower() == 'windows'):
+ shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath
+ shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
+ shellCmd += "> nul 2>&1 &"
+ else:
+ shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
+ shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
+ shellCmd += "> /dev/null 2>&1 &"
+ tdLog.info(shellCmd)
+ os.system(shellCmd)
+
+ def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'):
+ while 1:
+ tdSql.query("select * from %s.notifyinfo"%cdbName)
+ #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+ if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0):
+ break
+ else:
+ time.sleep(0.1)
+ return
+
+ def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'):
+ while 1:
+ tdSql.query("select * from %s.notifyinfo"%cdbName)
+ #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+ if tdSql.getRows() == 2 :
+ print(tdSql.getData(0, 1), tdSql.getData(1, 1))
+ if tdSql.getData(1, 1) == 1:
+ break
+ time.sleep(0.1)
+ return
+
+ def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1):
+ if dropFlag == 1:
+ tsql.execute("drop database if exists %s"%(dbName))
+
+ tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica))
+ tdLog.debug("complete to create database %s"%(dbName))
+ return
+
+ def create_stable(self,tsql, dbName,stbName):
+ tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName))
+ tdLog.debug("complete to create %s.%s" %(dbName, stbName))
+ return
+
+ def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1):
+ tsql.execute("use %s" %dbName)
+ pre_create = "create table"
+ sql = pre_create
+ #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
+ for i in range(ctbNum):
+ tagValue = 'beijing'
+ if (i % 2 == 0):
+ tagValue = 'shanghai'
+
+ sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue)
+ if (i > 0) and (i%100 == 0):
+ tsql.execute(sql)
+ sql = pre_create
+ if sql != pre_create:
+ tsql.execute(sql)
+
+ tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
+ return
+
+ def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ if startTs is None:
+ t = time.time()
+ startTs = int(round(t * 1000))
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ for i in range(ctbNum):
+ sql += " %s%d values "%(stbName,i)
+ for j in range(rowsPerTbl):
+ sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
+ if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s%d values " %(stbName,i)
+ else:
+ sql = "insert into "
+ #end sql
+ if sql != pre_insert:
+ #print("insert sql:%s"%sql)
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ t = time.time()
+ startTs = int(round(t * 1000))
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ for i in range(ctbNum):
+ sql += " %s%d values "%(ctbPrefix,i)
+ for j in range(rowsPerTbl):
+ if (j % 2 == 0):
+ sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j)
+ else:
+ sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j)
+ if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s%d values " %(ctbPrefix,i)
+ else:
+ sql = "insert into "
+ #end sql
+ if sql != pre_insert:
+ #print("insert sql:%s"%sql)
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ if startTs == 0:
+ t = time.time()
+ startTs = int(round(t * 1000))
+
+ ctbDict = {}
+ for i in range(ctbNum):
+ ctbDict[i] = 0
+
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ rowsOfCtb = 0
+ while rowsOfCtb < rowsPerTbl:
+ for i in range(ctbNum):
+ sql += " %s.%s_%d values "%(dbName,ctbPrefix,i)
+ for k in range(batchNum):
+ sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i])
+ ctbDict[i] += 1
+ if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl):
+ tsql.execute(sql)
+ sql = "insert into "
+ break
+ rowsOfCtb = ctbDict[0]
+
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+ tdLog.debug("start to insert data wiht auto create child table ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ if startTs == 0:
+ t = time.time()
+ startTs = int(round(t * 1000))
+
+ #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+ rowsOfSql = 0
+ for i in range(ctbNum):
+ sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i)
+ for j in range(rowsPerTbl):
+ sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
+ rowsOfSql += 1
+ if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ rowsOfSql = 0
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i)
+ else:
+ sql = "insert into "
+ #end sql
+ if sql != pre_insert:
+ #print("insert sql:%s"%sql)
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def syncCreateDbStbCtbInsertData(self, tsql, paraDict):
+ tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"])
+ tdCom.create_stable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdCom.create_ctable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ if "event" in paraDict and type(paraDict['event']) == type(threading.Event()):
+ paraDict["event"].set()
+
+ ctbPrefix = paraDict['ctbPrefix']
+ ctbNum = paraDict["ctbNum"]
+ for i in range(ctbNum):
+ tbName = '%s%s'%(ctbPrefix,i)
+ tdCom.insert_rows(tsql,dbname=paraDict["dbName"],tbname=tbName,start_ts_value=paraDict['startTs'],count=paraDict['rowsPerTbl'])
+ return
+
+ def threadFunction(self, **paraDict):
+ # create new connector for new tdSql instance in my thread
+ newTdSql = tdCom.newTdSql()
+ self.syncCreateDbStbCtbInsertData(self, newTdSql, paraDict)
+ return
+
+ def asyncCreateDbStbCtbInsertData(self, paraDict):
+ pThread = threading.Thread(target=self.threadFunction, kwargs=paraDict)
+ pThread.start()
+ return pThread
+
+
+ def close(self):
+ self.cursor.close()
+
+clusterComCreate = ClusterComCreate()
diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py
new file mode 100644
index 0000000000..6cf849d1b9
--- /dev/null
+++ b/tests/system-test/7-tmq/tmqCheckData1.py
@@ -0,0 +1,180 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+ #tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def checkFileContent(self, consumerId, queryString):
+ buildPath = tdCom.getBuildPath()
+ cfgPath = tdCom.getClientCfgPath()
+ dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
+ cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
+ tdLog.info(cmdStr)
+ os.system(cmdStr)
+
+ consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
+ tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
+
+ consumeFile = open(consumeRowsFile, mode='r')
+ queryFile = open(dstFile, mode='r')
+
+ # skip first line for it is schema
+ queryFile.readline()
+
+ while True:
+ dst = queryFile.readline()
+ src = consumeFile.readline()
+
+ if dst:
+ if dst != src:
+ tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
+ else:
+ break
+ return
+
+ def tmqCase1(self):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db1',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+
+ topicNameList = ['topic1', 'topic2', 'topic3']
+ expectRowsList = []
+ tmqCom.initConsumerTable()
+ tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+ tdLog.info("create stb")
+ tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdLog.info("create ctb")
+ tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ tdLog.info("insert data")
+ tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+
+ tdLog.info("create topics from stb with filter")
+ queryString = "select ts,c1,c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"])
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ # init consume info, and start tmq_sim, then check consume result
+ tdLog.info("insert consume info to consume processor")
+ consumerId = 0
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+ topicList = topicNameList[0]
+ ifcheckdata = 1
+ ifManualCommit = 1
+ keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+
+ if expectRowsList[0] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+ tdLog.exit("0 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ # reinit consume info, and start tmq_sim, then check consume result
+ tmqCom.initConsumerTable()
+ queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as database %s" %(topicNameList[1], paraDict['dbName'])
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ consumerId = 1
+ topicList = topicNameList[1]
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ if expectRowsList[1] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
+ tdLog.exit("1 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ # reinit consume info, and start tmq_sim, then check consume result
+ tmqCom.initConsumerTable()
+ queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ consumerId = 2
+ topicList = topicNameList[2]
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ if expectRowsList[2] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
+ tdLog.exit("2 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ time.sleep(10)
+ for i in range(len(topicNameList)):
+ tdSql.query("drop topic %s"%topicNameList[i])
+
+ tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+ def run(self):
+ tdSql.prepare()
+ self.tmqCase1()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py
index b8aa78e3ac..788ae3474c 100644
--- a/tests/system-test/7-tmq/tmqCommon.py
+++ b/tests/system-test/7-tmq/tmqCommon.py
@@ -86,7 +86,13 @@ class TMQCom:
shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
if (platform.system().lower() == 'windows'):
- shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath
+ processorName = buildPath + '\\build\\bin\\tmq_sim.exe'
+ if alias != 0:
+ processorNameNew = buildPath + '\\build\\bin\\tmq_sim_new.exe'
+ shellCmd = 'cp %s %s'%(processorName, processorNameNew)
+ os.system(shellCmd)
+ processorName = processorNameNew
+ shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
shellCmd += "> nul 2>&1 &"
else:
diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py
index 5b5658d528..bd8ec565d8 100644
--- a/tests/system-test/7-tmq/tmqError.py
+++ b/tests/system-test/7-tmq/tmqError.py
@@ -288,7 +288,10 @@ class TDTestCase:
tdLog.exit("tmq consume rows error!")
tdSql.query("drop topic %s"%topicFromStb1)
- os.system('pkill tmq_sim')
+ if (platform.system().lower() == 'windows'):
+ os.system("TASKKILL /F /IM tmq_sim.exe")
+ else:
+ os.system('pkill tmq_sim')
tdLog.printNoPrefix("======== test case 1 end ...... ")
diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py
new file mode 100644
index 0000000000..6b7e7375ff
--- /dev/null
+++ b/tests/system-test/7-tmq/tmqShow.py
@@ -0,0 +1,158 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+ #tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def tmqCase1(self):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db1',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 10,
+ 'rowsPerTbl': 4000,
+ 'batchNum': 15,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 20,
+ 'showMsg': 1,
+ 'showRow': 1}
+
+ topicNameList = ['topic1', 'topic2', 'topic3', 'topic4']
+ consumeGroupIdList = ['cgrp1', 'cgrp1', 'cgrp3', 'cgrp4']
+ consumerIdList = [0, 1, 2, 3]
+ tmqCom.initConsumerTable()
+ tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1)
+ tdLog.info("create stb")
+ tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdLog.info("create ctb")
+ tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ # tdLog.info("insert data")
+ # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+
+ tdLog.info("create 4 topics")
+ sqlString = "create topic %s as database %s" %(topicNameList[0], paraDict['dbName'])
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ sqlString = "create topic %s as stable %s.%s" %(topicNameList[1], paraDict['dbName'], paraDict['stbName'])
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ queryString = "select * from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s " %(topicNameList[3], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+
+ tdSql.query("show topics")
+ tdLog.debug(tdSql.queryResult)
+ rows = tdSql.getRows()
+ if rows != len(consumerIdList):
+ tdLog.exit("topic rows error")
+
+ for i in range (rows):
+ topicName = tdSql.getData(i,0)
+ matchFlag = 0
+ while matchFlag == 0:
+ for j in range(len(topicNameList)):
+ if topicName == topicNameList[j]:
+ matchFlag = 1
+ break
+ if matchFlag == 0:
+ tdLog.exit("topic name: %s is error", topicName)
+
+ # init consume info, and start tmq_sim, then check consume result
+ tdLog.info("insert consume info to consume processor")
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+ topicList = topicNameList[0]
+ ifcheckdata = 0
+ ifManualCommit = 0
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[0]
+ tmqCom.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ topicList = topicNameList[1]
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[1]
+ tmqCom.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ topicList = topicNameList[2]
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[2]
+ tmqCom.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ topicList = topicNameList[3]
+ keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[3]
+ tmqCom.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("async insert data")
+ pThread = tmqCom.asyncInsertData(paraDict)
+
+ time.sleep(5)
+ tdLog.info("check show consumers")
+ tdSql.query("show consumers")
+ # tdLog.info(tdSql.queryResult)
+ rows = tdSql.getRows()
+ tdLog.info("show consumers rows: %d"%rows)
+ if rows != len(topicNameList):
+ tdLog.exit("show consumers rows error")
+
+ tdLog.info("check show subscriptions")
+ tdSql.query("show subscriptions")
+ # tdLog.debug(tdSql.queryResult)
+ rows = tdSql.getRows()
+ tdLog.info("show subscriptions rows: %d"%rows)
+ if rows != paraDict['vgroups'] * len(topicNameList):
+ tdLog.exit("show subscriptions rows error")
+
+ pThread.join()
+
+ tdLog.info("insert process end, and start to check consume result")
+ expectRows = len(consumerIdList)
+ _ = tmqCom.selectConsumeResult(expectRows)
+
+ time.sleep(10)
+ for i in range(len(topicNameList)):
+ tdSql.query("drop topic %s"%topicNameList[i])
+
+ tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+ def run(self):
+ tdSql.prepare()
+ self.tmqCase1()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py
new file mode 100644
index 0000000000..ef74515792
--- /dev/null
+++ b/tests/system-test/99-TDcase/TD-16821.py
@@ -0,0 +1,186 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+ #tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def checkFileContent(self, consumerId, queryString):
+ buildPath = tdCom.getBuildPath()
+ cfgPath = tdCom.getClientCfgPath()
+ dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
+ cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
+ tdLog.info(cmdStr)
+ os.system(cmdStr)
+
+ consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
+ tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
+
+ consumeFile = open(consumeRowsFile, mode='r')
+ queryFile = open(dstFile, mode='r')
+
+ # skip first line for it is schema
+ queryFile.readline()
+
+ while True:
+ dst = queryFile.readline()
+ src = consumeFile.readline()
+
+ if dst:
+ if dst != src:
+ tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
+ else:
+ break
+ return
+
+ def tmqCase1(self):
+ tdLog.printNoPrefix("======== test case 1: ")
+ paraDict = {'dbName': 'db1',
+ 'dropFlag': 1,
+ 'event': '',
+ 'vgroups': 4,
+ 'stbName': 'stb',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+ 'ctbPrefix': 'ctb',
+ 'ctbNum': 1,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 10,
+ 'startTs': 1640966400000, # 2022-01-01 00:00:00.000
+ 'pollDelay': 10,
+ 'showMsg': 1,
+ 'showRow': 1}
+
+ topicNameList = ['topic1', 'topic2', 'topic3']
+ expectRowsList = []
+ tmqCom.initConsumerTable()
+ tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+ tdLog.info("create stb")
+ tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+ tdLog.info("create ctb")
+ tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+ tdLog.info("insert data")
+ tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+
+ tdLog.info("create topics from stb with filter")
+ # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+ # queryString = "select ts, c1, c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
+ queryString = "select * from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"])
+ # sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ # queryString = 'select * from %s.%s'%(paraDict["dbName"],paraDict["stbName"])
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ # init consume info, and start tmq_sim, then check consume result
+ tdLog.info("insert consume info to consume processor")
+ consumerId = 0
+ expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+ topicList = topicNameList[0]
+ ifcheckdata = 1
+ ifManualCommit = 1
+ keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+
+ if expectRowsList[0] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+ tdLog.exit("0 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ # reinit consume info, and start tmq_sim, then check consume result
+ tmqCom.initConsumerTable()
+
+ queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 3169" %(paraDict['dbName'], paraDict['stbName'])
+ sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ consumerId = 1
+ topicList = topicNameList[1]
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ if expectRowsList[1] != resultList[0]:
+ tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
+ tdLog.exit("1 tmq consume rows error!")
+
+ self.checkFileContent(consumerId, queryString)
+
+ # reinit consume info, and start tmq_sim, then check consume result
+ tmqCom.initConsumerTable()
+
+ queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+6137)
+ sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
+ tdLog.info("create topic sql: %s"%sqlString)
+ tdSql.execute(sqlString)
+ tdSql.query(queryString)
+ expectRowsList.append(tdSql.getRows())
+
+ consumerId = 2
+ topicList = topicNameList[2]
+ tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+ tdLog.info("start consume processor")
+ tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+ tdLog.info("wait the consume result")
+ expectRows = 1
+ resultList = tmqCom.selectConsumeResult(expectRows)
+ # if expectRowsList[2] != resultList[0]:
+ # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
+ # tdLog.exit("2 tmq consume rows error!")
+
+ # self.checkFileContent(consumerId, queryString)
+
+ time.sleep(10)
+ for i in range(len(topicNameList)):
+ tdSql.query("drop topic %s"%topicNameList[i])
+
+ tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+ def run(self):
+ tdSql.prepare()
+ self.tmqCase1()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat
deleted file mode 100644
index 59ddd3fb7d..0000000000
--- a/tests/system-test/fulltest.bat
+++ /dev/null
@@ -1,102 +0,0 @@
-
-python3 .\test.py -f 0-others\taosShell.py
-python3 .\test.py -f 0-others\taosShellError.py
-python3 .\test.py -f 0-others\taosShellNetChk.py
-python3 .\test.py -f 0-others\telemetry.py
-python3 .\test.py -f 0-others\taosdMonitor.py
-python3 .\test.py -f 0-others\udfTest.py
-python3 .\test.py -f 0-others\udf_create.py
-python3 .\test.py -f 0-others\udf_restart_taosd.py
-python3 .\test.py -f 0-others\cachelast.py
-
-python3 .\test.py -f 0-others\user_control.py
-python3 .\test.py -f 0-others\fsync.py
-
-python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py
-python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py
-python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py
-@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py
-python3 .\test.py -f 1-insert\alter_stable.py
-python3 .\test.py -f 1-insert\alter_table.py
-python3 .\test.py -f 2-query\between.py
-python3 .\test.py -f 2-query\distinct.py
-python3 .\test.py -f 2-query\varchar.py
-python3 .\test.py -f 2-query\ltrim.py
-python3 .\test.py -f 2-query\rtrim.py
-python3 .\test.py -f 2-query\length.py
-python3 .\test.py -f 2-query\char_length.py
-python3 .\test.py -f 2-query\upper.py
-python3 .\test.py -f 2-query\lower.py
-python3 .\test.py -f 2-query\join.py
-python3 .\test.py -f 2-query\join2.py
-python3 .\test.py -f 2-query\cast.py
-python3 .\test.py -f 2-query\union.py
-python3 .\test.py -f 2-query\union1.py
-python3 .\test.py -f 2-query\concat.py
-python3 .\test.py -f 2-query\concat2.py
-python3 .\test.py -f 2-query\concat_ws.py
-python3 .\test.py -f 2-query\concat_ws2.py
-python3 .\test.py -f 2-query\check_tsdb.py
-python3 .\test.py -f 2-query\spread.py
-python3 .\test.py -f 2-query\hyperloglog.py
-
-python3 .\test.py -f 2-query\timezone.py
-python3 .\test.py -f 2-query\Now.py
-python3 .\test.py -f 2-query\Today.py
-python3 .\test.py -f 2-query\max.py
-python3 .\test.py -f 2-query\min.py
-python3 .\test.py -f 2-query\count.py
-python3 .\test.py -f 2-query\last.py
-python3 .\test.py -f 2-query\first.py
-python3 .\test.py -f 2-query\To_iso8601.py
-python3 .\test.py -f 2-query\To_unixtimestamp.py
-python3 .\test.py -f 2-query\timetruncate.py
-python3 .\test.py -f 2-query\diff.py
-python3 .\test.py -f 2-query\Timediff.py
-
-python3 .\test.py -f 2-query\top.py
-python3 .\test.py -f 2-query\bottom.py
-python3 .\test.py -f 2-query\percentile.py
-python3 .\test.py -f 2-query\apercentile.py
-python3 .\test.py -f 2-query\abs.py
-python3 .\test.py -f 2-query\ceil.py
-python3 .\test.py -f 2-query\floor.py
-python3 .\test.py -f 2-query\round.py
-python3 .\test.py -f 2-query\log.py
-python3 .\test.py -f 2-query\pow.py
-python3 .\test.py -f 2-query\sqrt.py
-python3 .\test.py -f 2-query\sin.py
-python3 .\test.py -f 2-query\cos.py
-python3 .\test.py -f 2-query\tan.py
-python3 .\test.py -f 2-query\arcsin.py
-python3 .\test.py -f 2-query\arccos.py
-python3 .\test.py -f 2-query\arctan.py
-python3 .\test.py -f 2-query\query_cols_tags_and_or.py
-@REM # python3 .\test.py -f 2-query\nestedQuery.py
-@REM # TD-15983 subquery output duplicate name column.
-@REM # Please Xiangyang Guo modify the following script
-@REM # python3 .\test.py -f 2-query\nestedQuery_str.py
-
-python3 .\test.py -f 2-query\avg.py
-python3 .\test.py -f 2-query\elapsed.py
-python3 .\test.py -f 2-query\csum.py
-python3 .\test.py -f 2-query\mavg.py
-python3 .\test.py -f 2-query\diff.py
-python3 .\test.py -f 2-query\sample.py
-python3 .\test.py -f 2-query\function_diff.py
-python3 .\test.py -f 2-query\unique.py
-python3 .\test.py -f 2-query\stateduration.py
-python3 .\test.py -f 2-query\function_stateduration.py
-python3 .\test.py -f 2-query\statecount.py
-
-python3 .\test.py -f 7-tmq\basic5.py
-python3 .\test.py -f 7-tmq\subscribeDb.py
-python3 .\test.py -f 7-tmq\subscribeDb0.py
-python3 .\test.py -f 7-tmq\subscribeDb1.py
-python3 .\test.py -f 7-tmq\subscribeStb.py
-python3 .\test.py -f 7-tmq\subscribeStb0.py
-python3 .\test.py -f 7-tmq\subscribeStb1.py
-python3 .\test.py -f 7-tmq\subscribeStb2.py
-python3 .\test.py -f 7-tmq\subscribeStb3.py
-python3 .\test.py -f 7-tmq\subscribeStb4.py
-python3 .\test.py -f 7-tmq\db.py
\ No newline at end of file
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index ef217b828f..2e2e390d12 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -23,6 +23,7 @@ python3 ./test.py -f 1-insert/alter_stable.py
python3 ./test.py -f 1-insert/alter_table.py
python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
python3 ./test.py -f 1-insert/table_comment.py
+python3 ./test.py -f 1-insert/table_param_ttl.py
python3 ./test.py -f 2-query/between.py
python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py
@@ -116,6 +117,10 @@ python3 ./test.py -f 2-query/function_null.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3
+# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py -N 5 -M 3
+# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3
+
# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
@@ -136,6 +141,8 @@ python3 ./test.py -f 7-tmq/tmqError.py
python3 ./test.py -f 7-tmq/schema.py
python3 ./test.py -f 7-tmq/stbFilter.py
python3 ./test.py -f 7-tmq/tmqCheckData.py
+python3 ./test.py -f 7-tmq/tmqCheckData1.py
python3 ./test.py -f 7-tmq/tmqUdf.py
#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5
python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
+python3 ./test.py -f 7-tmq/tmqShow.py
diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat
index 275cbeebbb..d4032cbc0d 100644
--- a/tests/system-test/test-all.bat
+++ b/tests/system-test/test-all.bat
@@ -2,23 +2,27 @@
SETLOCAL EnableDelayedExpansion
for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a")
set /a a=0
-if %1 == full (
+if "%1" == "full" (
echo Windows Taosd Full Test
set /a exitNum=0
del /Q /F failed.txt
- set caseFile="fulltest.bat"
+ set caseFile="fulltest.sh"
if not "%2" == "" (
set caseFile="%2"
)
for /F "usebackq tokens=*" %%i in (!caseFile!) do (
- for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" (
- set /a a+=1
- echo !a! Processing %%i
- call :GetTimeSeconds !time!
- set time1=!_timeTemp!
- echo Start at !time!
- call %%i ARG1 > result_!a!.txt 2>error_!a!.txt
- if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+ call :CheckSkipCase %%i
+ if !skipCase! == false (
+ set line=%%i
+ if "!line:~,7!" == "python3" (
+ set /a a+=1
+ echo !a! Processing %%i
+ call :GetTimeSeconds !time!
+ set time1=!_timeTemp!
+ echo Start at !time!
+ call %%i ARG1 > result_!a!.txt 2>error_!a!.txt
+ if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+ )
)
)
exit !exitNum!
@@ -83,3 +87,8 @@ for %%a in (%tt%) do (
)
set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
goto :eof
+
+:CheckSkipCase
+set skipCase=false
+if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=true )
+:goto eof
\ No newline at end of file
diff --git a/tests/system-test/test.py b/tests/system-test/test.py
index 8a8356449c..76b83da348 100644
--- a/tests/system-test/test.py
+++ b/tests/system-test/test.py
@@ -22,8 +22,6 @@ import json
import platform
import socket
import threading
-from distutils.log import warn as printf
-from fabric2 import Connection
sys.path.append("../pytest")
from util.log import *
from util.dnodes import *
@@ -187,9 +185,9 @@ if __name__ == "__main__":
tdLog.info("Procedures for tdengine deployed in %s" % (host))
if platform.system().lower() == 'windows':
+ fileName = fileName.replace("/", os.sep)
if (masterIp == "" and not fileName[0:12] == "0-others\\udf"):
threading.Thread(target=checkRunTimeError,daemon=True).start()
- tdCases.logSql(logSql)
tdLog.info("Procedures for testing self-deployment")
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
@@ -208,18 +206,46 @@ if __name__ == "__main__":
uModule = importlib.import_module(moduleName)
try:
ucase = uModule.TDTestCase()
- if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)):
+ if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')):
updateCfgDict = ucase.updatecfgDict
updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode()
except Exception as r:
print(r)
else:
pass
- tdDnodes.deploy(1,updateCfgDict)
- tdDnodes.start(1)
- conn = taos.connect(
- host="%s"%(host),
- config=tdDnodes.sim.getCfgDir())
+ if dnodeNums == 1 :
+ tdDnodes.deploy(1,updateCfgDict)
+ tdDnodes.start(1)
+ tdCases.logSql(logSql)
+ else :
+ tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
+ dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)
+ tdDnodes = ClusterDnodes(dnodeslist)
+ tdDnodes.init(deployPath, masterIp)
+ tdDnodes.setTestCluster(testCluster)
+ tdDnodes.setValgrind(valgrind)
+ tdDnodes.stopAll()
+ for dnode in tdDnodes.dnodes:
+ tdDnodes.deploy(dnode.index,{})
+ for dnode in tdDnodes.dnodes:
+ tdDnodes.starttaosd(dnode.index)
+ tdCases.logSql(logSql)
+ conn = taos.connect(
+ host,
+ config=tdDnodes.getSimCfgPath())
+ print(tdDnodes.getSimCfgPath(),host)
+ cluster.create_dnode(conn)
+ try:
+ if cluster.check_dnode(conn) :
+ print("check dnode ready")
+ except Exception as r:
+ print(r)
+ if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True:
+ conn = None
+ else:
+ conn = taos.connect(
+ host="%s"%(host),
+ config=tdDnodes.sim.getCfgDir())
if is_test_framework:
tdCases.runOneWindows(conn, fileName)
else:
@@ -307,4 +333,5 @@ if __name__ == "__main__":
tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
else:
tdLog.info("not need to query")
- conn.close()
+ if conn is not None:
+ conn.close()
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index c2fba52e6e..81fa72d15a 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -635,8 +635,9 @@ void loop_consume(SThreadInfo* pInfo) {
}
}
+ int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000);
while (running) {
- TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);
+ TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay);
if (tmqMsg) {
if (0 != g_stConfInfo.showMsgFlag) {
totalRows += msg_process(tmqMsg, pInfo, totalMsgs);
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 8a017d378d..7b275c5a15 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -156,6 +156,7 @@ void shellRunSingleCommandImp(char *command) {
}
fname = sptr + 2;
+ while (*fname == ' ') fname++;
*sptr = '\0';
}
@@ -858,9 +859,7 @@ void shellGetGrantInfo() {
int32_t code = taos_errno(tres);
if (code != TSDB_CODE_SUCCESS) {
- if (code == TSDB_CODE_OPS_NOT_SUPPORT) {
- fprintf(stdout, "Server is Community Edition, %s\n\n", sinfo);
- } else {
+ if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS) {
fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", code, taos_errstr(tres));
}
return;
diff --git a/tools/taos-tools b/tools/taos-tools
index 28a49b447f..a875a057d1 160000
--- a/tools/taos-tools
+++ b/tools/taos-tools
@@ -1 +1 @@
-Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd
+Subproject commit a875a057d1225d85c6323b9edaccc2b1a9641987