diff --git a/Jenkinsfile2 b/Jenkinsfile2 index f582461fb2..f618912295 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -338,6 +338,14 @@ pipeline { changeRequest() } steps { + script { + def linux_node_ip = sh ( + script: 'ip addr|grep 192|grep -v virbr|awk "{print \\\$2}"|sed "s/\\/.*//"', + returnStdout: true + ).trim() + echo "${linux_node_ip}" + echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" + } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 120, unit: 'MINUTES'){ pre_test() diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index d7d472eb98..44651c0496 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -22,7 +22,7 @@ A complete TDengine system runs on one or more physical nodes. Logically, it inc **Virtual node (vnode)**: To better support data sharding, load balancing and prevent data from overheating or skewing, data nodes are virtualized into multiple virtual nodes (vnode, V2, V3, V4, etc. in the figure). Each vnode is a relatively independent work unit, which is the basic unit of time-series data storage and has independent running threads, memory space and persistent storage path. A vnode contains a certain number of tables (data collection points). When a new table is created, the system checks whether a new vnode needs to be created. The number of vnodes that can be created on a data node depends on the capacity of the hardware of the physical node where the data node is located. A vnode belongs to only one DB, but a DB can have multiple vnodes. In addition to the stored time-series data, a vnode also stores the schema and tag values of the included tables. A virtual node is uniquely identified in the system by the EP of the data node and the VGroup ID to which it belongs and is created and managed by the management node. -**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 5) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. +**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 3) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. **Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed in a leader/follower mechanism. Write operations can only be performed on the leader vnode, and then replicated to follower vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `“replica”` when creating a DB, and the default is 1. Using the multi-replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID. If two virtual nodes have the same vnode group ID, it means that they belong to the same group and the data is backed up to each other. The number of virtual nodes in a virtual node group can be dynamically changed, allowing only one, that is, no data replication. VGroup ID is never changed. Even if a virtual node group is deleted, its ID will not be reused. diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md index 433cb4808b..507ccea629 100644 --- a/docs/zh/21-tdinternal/01-arch.md +++ b/docs/zh/21-tdinternal/01-arch.md @@ -23,7 +23,7 @@ TDengine 分布式架构的逻辑结构图如下： **虚拟节点（vnode）：** 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点（vnode，图中 V2，V3，V4 等）。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个 DB，但一个 DB 可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的 schema、标签值等。一个虚拟节点由所属的数据节点的 EP，以及所属的 VGroup ID 在系统内唯一标识，由管理节点创建并管理。 -**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（开源版最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 +**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 **虚拟节点组（VGroup）：** 不同数据节点上的 vnode 可以组成一个虚拟节点组（vgroup）来保证系统的高可靠。虚拟节点组内采取 master/slave 的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个 DB 的副本数为 N，系统必须有至少 N 数据节点。副本数在创建 DB 时通过参数 replica 可以指定，缺省为 1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的 ID，VGroup ID。如果两个虚拟节点的 VGroup ID 相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID 是永远不变的，即使一个虚拟节点组被删除，它的 ID 也不会被收回重复利用。 diff --git a/include/client/taos.h b/include/client/taos.h index 5e7f12de0a..d31d5c582c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -128,6 +128,7 @@ typedef struct setConfRet { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f9ede63f7f..8f7f22a6a0 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -78,7 +78,6 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove -void debugCheckTags(STag *pTag); // TODO: remove // STRUCT ================= struct STColumn { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5688af18a..c5b0b89311 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1886,7 +1886,7 @@ typedef struct SVCreateStbReq { int8_t rollup; SSchemaWrapper schemaRow; SSchemaWrapper schemaTag; - SRSmaParam pRSmaParam; + SRSmaParam rsmaParam; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h new file mode 100644 index 0000000000..5e84b87a81 --- /dev/null +++ b/include/libs/function/taosudf.h @@ -0,0 +1,266 @@ +/* +* Copyright (c) 2019 TAOS Data, Inc. +* +* This program is free software: you can use, redistribute, and/or modify +* it under the terms of the GNU Affero General Public License, version 3 +* or later ("AGPL"), as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +*/ + +#ifndef TDENGINE_TAOSUDF_H +#define TDENGINE_TAOSUDF_H + +#include +#include +#include +#include + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__GNUC__) +#define FORCE_INLINE inline __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif +typedef struct SUdfColumnMeta { + int16_t type; + int32_t bytes; + uint8_t precision; + uint8_t scale; +} SUdfColumnMeta; + +typedef struct SUdfColumnData { + int32_t numOfRows; + int32_t rowsAlloc; + union { + struct { + int32_t nullBitmapLen; + char *nullBitmap; + int32_t dataLen; + char *data; + } fixLenCol; + + struct { + int32_t varOffsetsLen; + int32_t *varOffsets; + int32_t payloadLen; + char *payload; + int32_t payloadAllocLen; + } varLenCol; + }; +} SUdfColumnData; + + +typedef struct SUdfColumn { + SUdfColumnMeta colMeta; + bool hasNull; + SUdfColumnData colData; +} SUdfColumn; + +typedef struct SUdfDataBlock { + int32_t numOfRows; + int32_t numOfCols; + SUdfColumn **udfCols; +} SUdfDataBlock; + +typedef struct SUdfInterBuf { + int32_t bufLen; + char* buf; + int8_t numOfResult; //zero or one +} SUdfInterBuf; +typedef void *UdfcFuncHandle; + +// dynamic lib init and destroy +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); + +#define UDF_MEMORY_EXP_GROWTH 1.5 +#define NBIT (3u) +#define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) +#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) +#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) + +#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1) +#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row)))) +#define udfColDataSetNull_f(pColumn, row) \ + do { \ + BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \ + } while (0) + +#define udfColDataSetNotNull_f(pColumn, r_) \ + do { \ + BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_))); \ + } while (0) +#define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1) + +typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) +#define IS_VAR_DATA_TYPE(t) \ + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) +#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) + + +static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) { + if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { + return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row]; + } else { + return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row; + } +} + +static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) { + if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { + if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) { + if (udfColDataIsNull_var(pColumn, row)) { + return true; + } + char* data = udfColDataGetData(pColumn, row); + return (*data == TSDB_DATA_TYPE_NULL); + } else { + return udfColDataIsNull_var(pColumn, row); + } + } else { + return udfColDataIsNull_f(pColumn, row); + } +} + +static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + + if (newCapacity== 0 || newCapacity <= data->rowsAlloc) { + return TSDB_CODE_SUCCESS; + } + + int allocCapacity = (data->rowsAlloc< 8) ? 8 : data->rowsAlloc; + while (allocCapacity < newCapacity) { + allocCapacity *= UDF_MEMORY_EXP_GROWTH; + } + + if (IS_VAR_DATA_TYPE(meta->type)) { + char* tmp = (char*)realloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->varLenCol.varOffsets = (int32_t*)tmp; + data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; + // for payload, add data in udfColDataAppend + } else { + char* tmp = (char*)realloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->fixLenCol.nullBitmap = tmp; + data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); + if (meta->type == TSDB_DATA_TYPE_NULL) { + return TSDB_CODE_SUCCESS; + } + + tmp = (char*)realloc(data->fixLenCol.data, allocCapacity* meta->bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->fixLenCol.data = tmp; + data->fixLenCol.dataLen = allocCapacity* meta->bytes; + } + + data->rowsAlloc = allocCapacity; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) { + udfColEnsureCapacity(pColumn, row+1); + if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { + udfColDataSetNull_var(pColumn, row); + } else { + udfColDataSetNull_f(pColumn, row); + } + pColumn->hasNull = true; +} + +static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + udfColEnsureCapacity(pColumn, currentRow+1); + bool isVarCol = IS_VAR_DATA_TYPE(meta->type); + if (isNull) { + udfColDataSetNull(pColumn, currentRow); + } else { + if (!isVarCol) { + udfColDataSetNotNull_f(pColumn, currentRow); + memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); + } else { + int32_t dataLen = varDataTLen(pData); + if (meta->type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + sizeof(char)); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = sizeof(int64_t); + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = sizeof(char); + } + dataLen += sizeof(char); + } + + if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { + uint32_t newSize = data->varLenCol.payloadAllocLen; + if (newSize <= 1) { + newSize = 8; + } + + while (newSize < data->varLenCol.payloadLen + dataLen) { + newSize = newSize * UDF_MEMORY_EXP_GROWTH; + } + + char *buf = (char*)realloc(data->varLenCol.payload, newSize); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->varLenCol.payload = buf; + data->varLenCol.payloadAllocLen = newSize; + } + + uint32_t len = data->varLenCol.payloadLen; + data->varLenCol.varOffsets[currentRow] = len; + + memcpy(data->varLenCol.payload + len, pData, dataLen); + data->varLenCol.payloadLen += dataLen; + } + } + data->numOfRows = (currentRow + 1 > data->numOfRows) ? (currentRow+1) : data->numOfRows; + return 0; +} + +typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); + +typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); +typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf); +typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TAOSUDF_H diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 28b1fbe8ce..b4c05fea87 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -16,6 +16,13 @@ #ifndef TDENGINE_TUDF_H #define TDENGINE_TUDF_H +#undef malloc +#define malloc malloc +#undef free +#define free free +#undef realloc +#define alloc alloc +#include #include #include @@ -36,56 +43,6 @@ extern "C" { #endif #define UDF_DNODE_ID_ENV_NAME "DNODE_ID" -//====================================================================================== -//begin API to taosd and qworker - -typedef struct SUdfColumnMeta { - int16_t type; - int32_t bytes; - uint8_t precision; - uint8_t scale; -} SUdfColumnMeta; - -typedef struct SUdfColumnData { - int32_t numOfRows; - int32_t rowsAlloc; - union { - struct { - int32_t nullBitmapLen; - char *nullBitmap; - int32_t dataLen; - char *data; - } fixLenCol; - - struct { - int32_t varOffsetsLen; - int32_t *varOffsets; - int32_t payloadLen; - char *payload; - int32_t payloadAllocLen; - } varLenCol; - }; -} SUdfColumnData; - - -typedef struct SUdfColumn { - SUdfColumnMeta colMeta; - bool hasNull; - SUdfColumnData colData; -} SUdfColumn; - -typedef struct SUdfDataBlock { - int32_t numOfRows; - int32_t numOfCols; - SUdfColumn **udfCols; -} SUdfDataBlock; - -typedef struct SUdfInterBuf { - int32_t bufLen; - char* buf; - int8_t numOfResult; //zero or one -} SUdfInterBuf; -typedef void *UdfcFuncHandle; //low level APIs /** @@ -127,177 +84,6 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); int32_t cleanUpUdfs(); -// end API to taosd and qworker -//============================================================================================================================= -// begin API to UDF writer. - -// dynamic lib init and destroy -typedef int32_t (*TUdfInitFunc)(); -typedef int32_t (*TUdfDestroyFunc)(); - -//TODO: add API to check function arguments type, number etc. - -#define UDF_MEMORY_EXP_GROWTH 1.5 - -#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1) -#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row)))) -#define udfColDataSetNull_f(pColumn, row) \ - do { \ - BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \ - } while (0) - -#define udfColDataSetNotNull_f(pColumn, r_) \ - do { \ - BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_))); \ - } while (0) -#define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1) - - -static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) { - if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { - return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row]; - } else { - return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row; - } -} - -static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) { - if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { - if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) { - if (udfColDataIsNull_var(pColumn, row)) { - return true; - } - char* data = udfColDataGetData(pColumn, row); - return (*data == TSDB_DATA_TYPE_NULL); - } else { - return udfColDataIsNull_var(pColumn, row); - } - } else { - return udfColDataIsNull_f(pColumn, row); - } -} - -static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) { - SUdfColumnMeta *meta = &pColumn->colMeta; - SUdfColumnData *data = &pColumn->colData; - - if (newCapacity== 0 || newCapacity <= data->rowsAlloc) { - return TSDB_CODE_SUCCESS; - } - - int allocCapacity = TMAX(data->rowsAlloc, 8); - while (allocCapacity < newCapacity) { - allocCapacity *= UDF_MEMORY_EXP_GROWTH; - } - - if (IS_VAR_DATA_TYPE(meta->type)) { - char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - data->varLenCol.varOffsets = (int32_t*)tmp; - data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; - // for payload, add data in udfColDataAppend - } else { - char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - data->fixLenCol.nullBitmap = tmp; - data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); - if (meta->type == TSDB_DATA_TYPE_NULL) { - return TSDB_CODE_SUCCESS; - } - - tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - data->fixLenCol.data = tmp; - data->fixLenCol.dataLen = allocCapacity* meta->bytes; - } - - data->rowsAlloc = allocCapacity; - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) { - udfColEnsureCapacity(pColumn, row+1); - if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { - udfColDataSetNull_var(pColumn, row); - } else { - udfColDataSetNull_f(pColumn, row); - } - pColumn->hasNull = true; -} - -static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { - SUdfColumnMeta *meta = &pColumn->colMeta; - SUdfColumnData *data = &pColumn->colData; - udfColEnsureCapacity(pColumn, currentRow+1); - bool isVarCol = IS_VAR_DATA_TYPE(meta->type); - if (isNull) { - udfColDataSetNull(pColumn, currentRow); - } else { - if (!isVarCol) { - colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow); - memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); - } else { - int32_t dataLen = varDataTLen(pData); - if (meta->type == TSDB_DATA_TYPE_JSON) { - if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = 0; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = LONG_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { - dataLen = CHAR_BYTES; - } - dataLen += CHAR_BYTES; - } - - if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { - uint32_t newSize = data->varLenCol.payloadAllocLen; - if (newSize <= 1) { - newSize = 8; - } - - while (newSize < data->varLenCol.payloadLen + dataLen) { - newSize = newSize * UDF_MEMORY_EXP_GROWTH; - } - - char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize); - if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - data->varLenCol.payload = buf; - data->varLenCol.payloadAllocLen = newSize; - } - - uint32_t len = data->varLenCol.payloadLen; - data->varLenCol.varOffsets[currentRow] = len; - - memcpy(data->varLenCol.payload + len, pData, dataLen); - data->varLenCol.payloadLen += dataLen; - } - } - data->numOfRows = TMAX(currentRow + 1, data->numOfRows); - return 0; -} - -typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); - -typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); -typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf); -typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData); - - -// end API to UDF writer -//======================================================================================================================= #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 52d9b6f810..5fa7eed40c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,9 +26,9 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 -#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF +#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index e51b20a404..5d892352d6 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -524,7 +524,7 @@ void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg); int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg); -int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); @@ -541,7 +541,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode* ths, SyncSnapshotRsp* pMsg); // ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); -typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index c2c1a3534d..2b8c6a895e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -69,7 +69,7 @@ typedef struct SRpcMsg { } SRpcMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); -typedef bool (*RpcRfp)(int32_t code); +typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 6770be6e46..581e688ccb 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,6 +29,9 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif +#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#define TAOS_CONSOLE_PROMPT_CONTINUE " -> " + typedef struct TdCmd *TdCmdPtr; TdCmdPtr taosOpenCmd(const char* cmd); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8af5945300..8565eea63b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -25,7 +25,7 @@ extern "C" { // clang-format off #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) - + #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) @@ -35,7 +35,7 @@ const char* terrstr(); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) - + #define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d7bf4b60f1..8e0556125a 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -84,9 +84,12 @@ void closeTransporter(STscObj *pTscObj) { rpcClose(pTscObj->pAppInfo->pTransporter); } -static bool clientRpcRfp(int32_t code) { +static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index bbd477fa3b..52574dcc9f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,6 +81,19 @@ void taos_cleanup(void) { taosCloseLog(); } +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + // TODO: need re-implementation + return ret; +} + +setConfRet taos_set_config(const char *config){ +// TODO pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); +// pthread_mutex_unlock(&setConfMutex); + return ret; +} + TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8460a27a0e..7c1b31b6e4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -862,21 +862,6 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { printf("\n"); } -void debugCheckTags(STag *pTag) { - switch (pTag->flags) { - case 0x0: - case 0x20: - case 0x40: - case 0x60: - break; - default: - ASSERT(0); - } - - ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0); - ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver -} - static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { int32_t n = 0; @@ -999,7 +984,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { debugPrintSTag(*ppTag, __func__, __LINE__); #endif - debugCheckTags(*ppTag); // TODO: remove this line after debug return code; _err: diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9db5019512..e9b5c67d76 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3696,7 +3696,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1; if (pReq->isTsma) { - if (tDecodeBinaryAlloc(&decoder, &pReq->pTsma, NULL) < 0) return -1; + if (tDecodeBinary(&decoder, (uint8_t **)&pReq->pTsma, NULL) < 0) return -1; } tEndDecode(&decoder); @@ -3707,9 +3707,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { taosArrayDestroy(pReq->pRetensions); pReq->pRetensions = NULL; - if (pReq->isTsma) { - taosMemoryFreeClear(pReq->pTsma); - } return 0; } @@ -4747,9 +4744,8 @@ int32_t tDecodeSRSmaParam(SDecoder *pCoder, SRSmaParam *pRSmaParam) { if (tDecodeI64v(pCoder, &pRSmaParam->watermark[i]) < 0) return -1; if (tDecodeI32v(pCoder, &pRSmaParam->qmsgLen[i]) < 0) return -1; if (pRSmaParam->qmsgLen[i] > 0) { - uint64_t len; - if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg[i], &len) < 0) - return -1; // qmsgLen contains len of '\0' + tDecoderMalloc(pCoder, pRSmaParam->qmsgLen[i]); + if (tDecodeBinary(pCoder, (uint8_t **)&pRSmaParam->qmsg[i], NULL) < 0) return -1; // qmsgLen contains len of '\0' } else { pRSmaParam->qmsg[i] = NULL; } @@ -4767,7 +4763,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { - if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1; + if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } tEndEncode(pCoder); @@ -4783,7 +4779,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { - if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1; + if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } tEndDecode(pCoder); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index a4745abd5b..7e31cc3144 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -248,9 +248,12 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { } } -static bool rpcRfp(int32_t code) { +static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index bc2d5c82b1..1497bba11c 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -39,8 +39,10 @@ typedef struct { int32_t id; int32_t errCode; int32_t acceptableCode; - ETrnStage stage; + int32_t retryCode; ETrnAct actionType; + ETrnStage stage; + int8_t reserved; int8_t rawWritten; int8_t msgSent; int8_t msgReceived; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 4c9974ba1a..b66082dd98 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -442,7 +442,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); @@ -491,7 +491,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 029a1e6f8c..164bcc7d60 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -234,7 +234,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (retrieveReq.user[0] != 0) { memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); } else { - memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN); + memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); } if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 92f85ecd04..dd01a0fa16 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -427,17 +427,17 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.schemaTag.pSchema = pStb->pTags; if (req.rollup) { - req.pRSmaParam.maxdelay[0] = pStb->maxdelay[0]; - req.pRSmaParam.maxdelay[1] = pStb->maxdelay[1]; + req.rsmaParam.maxdelay[0] = pStb->maxdelay[0]; + req.rsmaParam.maxdelay[1] = pStb->maxdelay[1]; if (pStb->ast1Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[0], &req.pRSmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[0]) < 0) { + if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) { goto _err; } } if (pStb->ast2Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[1], &req.pRSmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[1]) < 0) { + if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) { goto _err; } } @@ -470,12 +470,12 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt tEncoderClear(&encoder); *pContLen = contLen; - taosMemoryFreeClear(req.pRSmaParam.qmsg[0]); - taosMemoryFreeClear(req.pRSmaParam.qmsg[1]); + taosMemoryFreeClear(req.rsmaParam.qmsg[0]); + taosMemoryFreeClear(req.rsmaParam.qmsg[1]); return pHead; _err: - taosMemoryFreeClear(req.pRSmaParam.qmsg[0]); - taosMemoryFreeClear(req.pRSmaParam.qmsg[1]); + taosMemoryFreeClear(req.rsmaParam.qmsg[0]); + taosMemoryFreeClear(req.rsmaParam.qmsg[1]); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a9de1a05a7..b09ee5f608 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,15 +15,15 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 -#define TRANS_ARRAY_SIZE 8 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); @@ -55,7 +55,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); -static int32_t mndProcessTransReq(SRpcMsg *pReq); +static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTtl(SRpcMsg *pReq); static int32_t mndProcessKillTransReq(SRpcMsg *pReq); @@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTransActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans); @@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->startFunc = 0; } return 0; } static void mndTransDropData(STrans *pTrans) { - mndTransDropActions(pTrans->redoActions); - mndTransDropActions(pTrans->undoActions); - mndTransDropActions(pTrans->commitActions); + if (pTrans->redoActions != NULL) { + mndTransDropActions(pTrans->redoActions); + pTrans->redoActions = NULL; + } + if (pTrans->undoActions != NULL) { + mndTransDropActions(pTrans->undoActions); + pTrans->undoActions = NULL; + } + if (pTrans->commitActions != NULL) { + mndTransDropActions(pTrans->commitActions); + pTrans->commitActions = NULL; + } if (pTrans->rpcRsp != NULL) { taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; @@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->stopFunc = 0; } mndTransDropData(pTrans); @@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) { if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } @@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->errCode = pRsp->code; } - mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action, - pRsp->code, pAction->acceptableCode); + mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, + mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); mndTransExecute(pMnode, pTrans); _OVER: @@ -884,6 +907,21 @@ _OVER: return 0; } +static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { + pAction->rawWritten = 0; + pAction->msgSent = 0; + pAction->msgReceived = 0; + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || + pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; + mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), + pAction->id, pAction->epSet.inUse); + } else { + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id); + } + pAction->errCode = 0; +} + static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { int32_t numOfActions = taosArrayGetSize(pArray); @@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) continue; if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue; - pAction->rawWritten = 0; - pAction->msgSent = 0; - pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { - pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; - mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), - action, pAction->epSet.inUse); - } else { - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); - } - pAction->errCode = 0; + mndTransResetAction(pMnode, pTrans, pAction); } } @@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; - pAction->msgSent = 0; - pAction->msgReceived = 0; - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + mndTransResetAction(pMnode, pTrans, pAction); + } else { + mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action); } } else { code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->rawWritten) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; + } else { + mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action); } } } @@ -1156,9 +1185,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; + } else if (code == pAction->retryCode) { + mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + taosMsleep(300); + action--; + continue; } else { terrno = code; pTrans->code = code; + mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, + mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); break; } } @@ -1343,7 +1379,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pMnode, pTrans); } -static int32_t mndProcessTransReq(SRpcMsg *pReq) { +static int32_t mndProcessTransTimer(SRpcMsg *pReq) { mndTransPullup(pReq->info.node); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0e931e0a9c..d42016e6a0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD action.pCont = pHead; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_CONFIRM; + // incorrect redirect result will cause this erro + action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pHead); @@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb action.contLen = contLen; action.msgType = TDMT_SYNC_SET_VNODE_STANDBY; action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + // Keep retrying until the target vnode is not the leader + action.retryCode = TSDB_CODE_SYN_IS_LEADER; if (isRedo) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1003,7 +1007,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId); if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; @@ -1017,10 +1021,19 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } + + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1168,10 +1181,19 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER; } - pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + { + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } + + { + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1229,7 +1251,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1351,7 +1374,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1424,13 +1448,25 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S } else { } - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } return 0; @@ -1538,12 +1574,23 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1; if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1; - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } + + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); } - sdbSetRawStatus(pRaw, SDB_STATUS_READY); mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1552,7 +1599,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD return 0; } -static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) { +static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst, + SHashObj *pBalancedVgroups) { void *pIter = NULL; int32_t code = -1; SSdb *pSdb = pMnode->pSdb; @@ -1561,6 +1609,10 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SVgObj *pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + if (taosHashGet(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t)) != NULL) { + sdbRelease(pSdb, pVgroup); + continue; + } bool existInSrc = false; bool existInDst = false; @@ -1577,6 +1629,9 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst); + if (code == 0) { + code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t)); + } mndReleaseDb(pMnode, pDb); sdbRelease(pSdb, pVgroup); sdbCancelFetch(pSdb, pIter); @@ -1590,6 +1645,10 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { int32_t code = -1; int32_t numOfVgroups = 0; STrans *pTrans = NULL; + SHashObj *pBalancedVgroups = NULL; + + pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pBalancedVgroups == NULL) goto _OVER; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; @@ -1613,18 +1672,18 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { - code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst); + code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups); if (code == 0) { pSrc->numOfVnodes--; pDst->numOfVnodes++; numOfVgroups++; continue; } else { - mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); - return -1; + mDebug("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + break; } } else { - mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id); + mDebug("trans:%d, no vgroup need to balance any more", pTrans->id); break; } } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 858376519d..5c2d2cd712 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -125,6 +125,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo bool isTsdbCacheLastRow(tsdbReaderT *pReader); int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); +int32_t tsdbGetStbIdList(SMeta *pMeta, int64_t suid, SArray *list); void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); @@ -198,15 +199,20 @@ typedef struct { uint64_t groupId; } STableKeyInfo; +#define TABLE_ROLLUP_ON ((int8_t)0x1) +#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) +#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) struct SMetaEntry { int64_t version; int8_t type; + int8_t flags; // TODO: need refactor? tb_uid_t uid; char *name; union { struct { SSchemaWrapper schemaRow; SSchemaWrapper schemaTag; + SRSmaParam rsmaParam; } stbEntry; struct { int64_t ctime; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 63a8e7ec68..66d1689d57 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -69,6 +69,7 @@ struct SMeta { TTB* pUidIdx; TTB* pNameIdx; TTB* pCtbIdx; + TTB* pSuidIdx; // ivt idx and idx void* pTagIvtIdx; TTB* pTagIdx; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1489b02492..4b42ab5263 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -60,8 +60,9 @@ struct SRSmaStat { SSma *pSma; void *tmrHandle; tmr_h tmrId; - int8_t tmrStat; int32_t tmrSeconds; + int8_t triggerStat; + int8_t runningStat; SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; }; @@ -72,13 +73,21 @@ struct SSmaStat { }; T_REF_DECLARE() }; -#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) -#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) -#define SMA_RSMA_INFO_HASH(s) ((s)->rsmaStat.rsmaInfoHash) -#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle) -#define SMA_RSMA_TMR_STAT(s) ((s)->rsmaStat.tmrStat) -#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define RSMA_TMR_ID(r) ((r)->tmrId) +#define RSMA_TMR_HANDLE(r) ((r)->tmrHandle) +#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) +#define RSMA_RUNNING_STAT(r) (&(r)->runningStat) +enum { + TASK_TRIGGER_STAT_INIT = 0, + TASK_TRIGGER_STAT_ACTIVE = 1, + TASK_TRIGGER_STAT_INACTIVE = 2, + TASK_TRIGGER_STAT_CANCELLED = 3, + TASK_TRIGGER_STAT_FINISHED = 4, +}; void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -167,13 +176,18 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); +void *tdFreeRSmaInfo(SRSmaInfo *pInfo); -void *tdFreeRSmaInfo(SRSmaInfo *pInfo); - +int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); +int32_t tdProcessRSmaRestoreImpl(SSma *pSma); int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +// smaFileUtil ================ + +#define TD_FILE_HEAD_SIZE 512 + typedef struct STFInfo STFInfo; typedef struct STFile STFile; @@ -181,7 +195,7 @@ struct STFInfo { uint32_t magic; uint32_t ftype; uint32_t fver; - uint64_t fsize; + int64_t fsize; }; struct STFile { @@ -199,11 +213,8 @@ struct STFile { #define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) #define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf)) #define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL) -#define TD_FILE_STATE(tf) ((tf)->state) #define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TD_FILE_DID(tf) (TD_FILE_F(tf)->did) -#define TD_FILE_IS_OK(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_OK) -#define TD_FILE_IS_BAD(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD) int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); @@ -212,12 +223,14 @@ int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); +int64_t tdGetTFileSize(STFile *pTFile, int64_t *size); int32_t tdRemoveTFile(STFile *pTFile); int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); + +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3b30224e17..68ed6dde51 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -76,6 +76,7 @@ void vnodeFree(void* p); // meta typedef struct SMCtbCursor SMCtbCursor; +typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta); @@ -97,6 +98,9 @@ int metaGetTbNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseCtbCursor(SMCtbCursor* pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseStbCursor(SMStbCursor* pStbCur); +tb_uid_t metaStbCursorNext(SMStbCursor* pStbCur); STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid); STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy); SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); @@ -158,6 +162,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool // sma int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); +int32_t smaCloseEnv(SSma* pSma); +int32_t smaCloseEx(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index acf5b0b613..23d7665ba3 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -24,22 +24,25 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeCStr(pCoder, pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { + if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (TABLE_IS_ROLLUP(pME->flags)) { + if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + } } else if (pME->type == TSDB_CHILD_TABLE) { if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; - if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0){ if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; } if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; - debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; } else if (pME->type == TSDB_NORMAL_TABLE) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; - if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0){ if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; } @@ -64,23 +67,26 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { + if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (TABLE_IS_ROLLUP(pME->flags)) { + if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + } } else if (pME->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0){ if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; } if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) - debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug } else if (pME->type == TSDB_NORMAL_TABLE) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0){ if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 95510c25e5..614ccc5b5e 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -92,6 +92,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } + // open pSuidIdx + ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx); + if (ret < 0) { + metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open pTagIdx // TODO(yihaoDeng), refactor later char indexFullPath[128] = {0}; @@ -141,6 +148,7 @@ _err: if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); @@ -162,6 +170,7 @@ int metaClose(SMeta *pMeta) { if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); #endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 910f4bba51..85106f46c2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -325,6 +325,72 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { return pCtbIdxKey->uid; } +struct SMStbCursor { + SMeta *pMeta; + TBC *pCur; + tb_uid_t suid; + void *pKey; + void *pVal; + int kLen; + int vLen; +}; + +SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) { + SMStbCursor *pStbCur = NULL; + int ret = 0; + int c = 0; + + pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur)); + if (pStbCur == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pStbCur->pMeta = pMeta; + pStbCur->suid = suid; + metaRLock(pMeta); + + ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL); + if (ret < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaULock(pMeta); + taosMemoryFree(pStbCur); + return NULL; + } + + // move to the suid + tdbTbcMoveTo(pStbCur->pCur, &suid, sizeof(suid), &c); + if (c > 0) { + tdbTbcMoveToNext(pStbCur->pCur); + } + + return pStbCur; +} + +void metaCloseStbCursor(SMStbCursor *pStbCur) { + if (pStbCur) { + if (pStbCur->pMeta) metaULock(pStbCur->pMeta); + if (pStbCur->pCur) { + tdbTbcClose(pStbCur->pCur); + + tdbFree(pStbCur->pKey); + tdbFree(pStbCur->pVal); + } + + taosMemoryFree(pStbCur); + } +} + +tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { + int ret; + + ret = tdbTbcNext(pStbCur->pCur, &pStbCur->pKey, &pStbCur->kLen, &pStbCur->pVal, &pStbCur->vLen); + if (ret < 0) { + return 0; + } + return *(tb_uid_t*)pStbCur->pKey; +} + STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { // SMetaReader mr = {0}; STSchema * pTSchema = NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7109bf1dfc..a621b4ddb0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -23,6 +23,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); @@ -138,6 +139,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { me.name = pReq->name; me.stbEntry.schemaRow = pReq->schemaRow; me.stbEntry.schemaTag = pReq->schemaTag; + if (pReq->rollup) { + TABLE_SET_ROLLUP(me.flags); + me.stbEntry.rsmaParam = pReq->rsmaParam; + } if (metaHandleEntry(pMeta, &me) < 0) goto _err; @@ -209,6 +214,7 @@ _drop_super_table: &pMeta->txn); tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); metaULock(pMeta); @@ -436,11 +442,13 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); + if (e.type == TSDB_CHILD_TABLE) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) } else if (e.type == TSDB_SUPER_TABLE) { + tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn); // drop schema.db (todo) } @@ -911,6 +919,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn); +} + static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } @@ -1081,6 +1093,10 @@ static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } else { // update schema.db if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err; + + if (pME->type == TSDB_SUPER_TABLE) { + if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err; + } } if (pME->type != TSDB_SUPER_TABLE) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4e907e93ca..7f115633b9 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -126,22 +126,21 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS } if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma; + SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); + pRSmaStat->pSma = (SSma *)pSma; // init timer - SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G"); - if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA"); + if (!RSMA_TMR_HANDLE(pRSmaStat)) { taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } - - atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE); // init hash - SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - if (!SMA_RSMA_INFO_HASH(*pSmaStat)) { - if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) { - taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat)); + if (!RSMA_INFO_HASH(pRSmaStat)) { + if (RSMA_TMR_HANDLE(pRSmaStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat)); } taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; @@ -155,12 +154,80 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS return TSDB_CODE_SUCCESS; } -static void *tdFreeTSmaStat(STSmaStat *pStat) { +static void tdDestroyTSmaStat(STSmaStat *pStat) { if (pStat) { + smaDebug("destroy tsma stat"); tDestroyTSma(pStat->pTSma); taosMemoryFreeClear(pStat->pTSma); - taosMemoryFreeClear(pStat); + taosMemoryFreeClear(pStat->pTSchema); } +} + +static void *tdFreeTSmaStat(STSmaStat *pStat) { + tdDestroyTSmaStat(pStat); + taosMemoryFreeClear(pStat); + return NULL; +} + +static void tdDestroyRSmaStat(SRSmaStat *pStat) { + if (pStat) { + smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma)); + // step 1: set persistence task cancelled + atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); + + // step 2: stop the persistence timer + taosTmrStopA(&RSMA_TMR_ID(pStat)); + + // step 3: wait the persistence thread to finish + int32_t nLoops = 0; + if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) { + while (1) { + if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) { + break; + } else { + smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat))); + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + } + + // step 4: destroy the rsma info and associated fetch tasks + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); + while (infoHash) { + SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash; + tdFreeRSmaInfo(pSmaInfo); + infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); + } + taosHashCleanup(RSMA_INFO_HASH(pStat)); + + // step 5: wait all triggered fetch tasks finished + nLoops = 0; + while (1) { + if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { + break; + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + // step 6: cleanup the timer handle + if (RSMA_TMR_HANDLE(pStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); + } + } +} + +static void *tdFreeRSmaStat(SRSmaStat *pStat) { + tdDestroyRSmaStat(pStat); + taosMemoryFreeClear(pStat); return NULL; } @@ -179,19 +246,9 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - tdFreeTSmaStat(&pSmaStat->tsmaStat); + tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - if (SMA_RSMA_TMR_HANDLE(pSmaStat)) { - taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat)); - } - // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL); - while (infoHash) { - SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; - tdFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash); - } - taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat)); + tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat)); } else { ASSERT(0); } @@ -259,35 +316,4 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { tdUnLockSma(pSma); return TSDB_CODE_SUCCESS; -}; - -int32_t smaTimerInit(void **timer, int8_t *initFlag, const char *label) { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(initFlag, 0, 2); - if (old != 2) break; - } - - if (old == 0) { - *timer = taosTmrInit(10000, 100, 10000, label); - if (!(*timer)) { - atomic_store_8(initFlag, 0); - return -1; - } - atomic_store_8(initFlag, 1); - } - return 0; -} - -void smaTimerCleanUp(void *timer, int8_t *initFlag) { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(initFlag, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - taosTmrCleanUp(timer); - atomic_store_8(initFlag, 0); - } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 383c264e7b..88ed7426f7 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -18,6 +18,7 @@ static int32_t smaEvalDays(SRetention *r, int8_t precision); static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type); +static int32_t rsmaRestore(SSma *pSma); #define SMA_SET_KEEP_CFG(l) \ do { \ @@ -100,6 +101,9 @@ int32_t smaOpen(SVnode *pVnode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + + pVnode->pSma = pSma; + pSma->pVnode = pVnode; taosThreadMutexInit(&pSma->mutex, NULL); pSma->locked = false; @@ -117,37 +121,53 @@ int32_t smaOpen(SVnode *pVnode) { ASSERT(0); } } + + // restore the rsma +#if 0 + if (rsmaRestore(pSma) < 0) { + goto _err; + } +#endif } - pVnode->pSma = pSma; return 0; _err: - taosMemoryFreeClear(pSma); return -1; } -int32_t smaClose(SSma *pSma) { +int32_t smaCloseEnv(SSma *pSma) { + if (pSma) { + SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); + SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); + } + return 0; +} + +int32_t smaCloseEx(SSma *pSma) { if (pSma) { taosThreadMutexDestroy(&pSma->mutex); if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); - // SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); - // SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); taosMemoryFreeClear(pSma); } return 0; } +int32_t smaClose(SSma *pSma) { + smaCloseEnv(pSma); + smaCloseEx(pSma); + return 0; +} + /** * @brief rsma env restore - * - * @param pSma - * @return int32_t + * + * @param pSma + * @return int32_t */ -int32_t smaRestore(SSma *pSma) { - if (!pSma) return 0; - // iterate all stables to restore the rsma env - - return TSDB_CODE_SUCCESS; +static int32_t rsmaRestore(SSma *pSma) { + ASSERT(VND_IS_RSMA(pSma->pVnode)); + + return tdProcessRSmaRestoreImpl(pSma); } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 6d7b2d0053..ed5b6f4055 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -14,11 +14,15 @@ */ #include "sma.h" -#include "tstream.h" +#define RSMA_QTASKINFO_PERSIST_MS 7200000 +#define RSMA_QTASKINFO_BUFSIZE 32768 typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; +typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; +typedef struct SRSmaQTaskFIter SRSmaQTaskFIter; + static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle, @@ -27,17 +31,24 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType tb_uid_t suid, int8_t level); static void tdRSmaFetchTrigger(void *param, void *tmrId); static void tdRSmaPersistTrigger(void *param, void *tmrId); +static void *tdRSmaPersistExec(void *param); +static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName); + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile); +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish); +static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd); +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; void *taskInfo; // qTaskInfo_t - void *tmrHandle; tmr_h tmrId; int8_t level; int8_t tmrInitFlag; - int8_t triggerStatus; // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE + int8_t triggerStat; int32_t maxDelay; }; + struct SRSmaInfo { STSchema *pTSchema; SSma *pSma; @@ -45,11 +56,38 @@ struct SRSmaInfo { SRSmaInfoItem items[TSDB_RETENTION_L2]; }; -static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { +struct SRSmaQTaskInfoItem { + int32_t len; + int8_t type; + int64_t suid; + void *qTaskInfo; +}; + +struct SRSmaQTaskFIter { + STFile *pTFile; + int64_t offset; + int64_t fsize; + int32_t nBytes; + int32_t nAlloc; + char *buf; + // ------------ + int32_t nBufPos; +}; + +static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { + return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t); +} + +static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); } + +static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); qDestroyTask(otaskHandle); + } else { + smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } } @@ -58,14 +96,19 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { - tdFreeTaskHandle(pItem->taskInfo); - } - if (pItem->tmrHandle) { - taosTmrCleanUp(pItem->tmrHandle); + smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid, + pItem->tmrId, i + 1); + taosTmrStopA(&pItem->tmrId); + tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1); + } else { + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", + SMA_VID(pInfo->pSma), pInfo->suid, i + 1); } } taosMemoryFree(pInfo->pTSchema); taosMemoryFree(pInfo); + } else { + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid); } return NULL; @@ -81,9 +124,9 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; if (!suid || !tbUids) { @@ -92,27 +135,31 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA return TSDB_CODE_FAILED; } - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } - if (pRSmaInfo->items[0].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + if (pRSmaInfo->items[0].taskInfo) { + if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } } - if (pRSmaInfo->items[1].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + if (pRSmaInfo->items[1].taskInfo) { + if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } } return TSDB_CODE_SUCCESS; @@ -159,9 +206,9 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -195,11 +242,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo if (param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->pRsmaInfo = pRSmaInfo; - pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); + pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle); if (!pItem->taskInfo) { goto _err; } - pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); @@ -211,10 +258,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); - pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); - if (!pItem->tmrHandle) { - goto _err; - } } return TSDB_CODE_SUCCESS; _err: @@ -222,26 +265,21 @@ _err: } /** - * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * @brief for rsam create or restore * - * @param pTsdb - * @param pMeta - * @param pReq + * @param pSma + * @param param + * @param suid + * @param tbName * @return int32_t */ -int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { - SSma *pSma = pVnode->pSma; - if (!pReq->rollup) { - smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - SMeta *pMeta = pVnode->pMeta; - SMsgCb *pMsgCb = &pVnode->msgCb; - SRSmaParam *param = &pReq->pRSmaParam; +int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { + SVnode *pVnode = pSma->pVnode; + SMeta *pMeta = pVnode->pMeta; + SMsgCb *pMsgCb = &pVnode->msgCb; if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { - smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -251,13 +289,13 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { } SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally - smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -281,26 +319,33 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { .vnode = pVnode, }; - STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), pReq->suid, -1); + STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; goto _err; } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->pSma = pSma; - pRSmaInfo->suid = pReq->suid; + pRSmaInfo->suid = suid; if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { goto _err; } + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) { goto _err; } - if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { - smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); + smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid); + } + + // start the persist timer + if (TASK_TRIGGER_STAT_INIT == + atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) { + taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat)); } return TSDB_CODE_SUCCESS; @@ -310,6 +355,24 @@ _err: return TSDB_CODE_FAILED; } +/** + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * + * @param pTsdb + * @param pMeta + * @param pReq + * @return int32_t + */ +int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { + SSma *pSma = pVnode->pSma; + if (!pReq->rollup) { + smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name); +} + /** * @brief store suid/[uids], prefer to use array and then hash * @@ -432,6 +495,16 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { return 0; } +static void tdDestroySDataBlockArray(SArray *pArray) { +#if 0 + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pArray, i); + blockDestroyInner(pDataBlock); + } +#endif + taosArrayDestroy(pArray); +} + static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) { SArray *pResult = NULL; SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo; @@ -466,6 +539,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) #endif STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; + // TODO: the schema update should be handled if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { goto _err; } @@ -476,18 +550,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } taosMemoryFreeClear(pReq); + } else if (terrno == 0) { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); } else { - smaDebug("vgId:%d, no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); } - if (blkType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - } - - taosArrayDestroy(pResult); + tdDestroySDataBlockArray(pResult); return TSDB_CODE_SUCCESS; _err: - taosArrayDestroy(pResult); + tdDestroySDataBlockArray(pResult); return TSDB_CODE_FAILED; } @@ -499,26 +571,38 @@ _err: */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; + SSma *pSma = pItem->pRsmaInfo->pSma; + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); - if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__, - taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); - SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); - - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); - } else { - smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__, - taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); + int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); + if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) { + smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma), + pItem->level, pItem->pRsmaInfo->suid); + return; } - // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + int8_t fetchTriggerStat = + atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) { + smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pItem->pRsmaInfo->suid); + + tdRefSmaStat(pSma, (SSmaStat *)pStat); + + SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); + + tdUnRefSmaStat(pSma, (SSmaStat *)pStat); + + } else { + smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pItem->pRsmaInfo->suid); + } } -static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, - tb_uid_t suid, int8_t level) { +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid, + int8_t level) { if (!pItem || !pItem->taskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_SUCCESS; @@ -533,14 +617,17 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId()); + atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma)); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); - taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle); - taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + if (pStat->tmrHandle) { + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId); + } else { + ASSERT(0); + } return TSDB_CODE_SUCCESS; } @@ -552,10 +639,10 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -608,7 +695,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } -void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) { +static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) { tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); } @@ -618,6 +705,11 @@ static void *tdRSmaPersistExec(void *param) { SSma *pSma = pRSmaStat->pSma; STfs *pTfs = pSma->pVnode->pTfs; int64_t toffset = 0; + bool isFileCreated = false; + + if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) { + goto _end; + } void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); if (!infoHash) { @@ -625,67 +717,143 @@ static void *tdRSmaPersistExec(void *param) { } STFile tFile = {0}; - int32_t vid = 2; - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); - tdInitTFile(&tFile, pTfs, qTaskInfoFName); - tdCreateTFile(&tFile, pTfs, true, -1); + int32_t vid = SMA_VID(pSma); while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - char *pOutput = NULL; - int32_t len = 0; - if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) { - smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid, - terrstr(terrno)); - } else { - smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid, - len); - } - tdAppendTFile(&tFile, &len, sizeof(len), &toffset); - tdAppendTFile(&tFile, pOutput, len, &toffset); - taosMemoryFree(pOutput); +#if 0 + smaDebug("table %" PRIi64 " sleep 15s start ...", pRSmaInfo->items[0].pRsmaInfo->suid); + for (int32_t i = 15; i > 0; --i) { + taosSsleep(1); + smaDebug("table %" PRIi64 " countdown %d", pRSmaInfo->items[0].pRsmaInfo->suid, i); + } + smaDebug("table %" PRIi64 " sleep 15s end ...", pRSmaInfo->items[0].pRsmaInfo->suid); +#endif + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; + if (!taskInfo) { + smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); + continue; + } + char *pOutput = NULL; + int32_t len = 0; + int8_t type = (int8_t)(i + 1); + if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { + smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1, + terrstr(terrno)); + goto _err; + } else { + if (!pOutput) { + smaDebug("vgId:%d, table %" PRIi64 + " level %d serialize rsma task success but no output(len %d) and no need to persist", + vid, pRSmaInfo->suid, i + 1, len); + continue; + } else if (len <= 0) { + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and no need to persist", + vid, pRSmaInfo->suid, i + 1, len); + taosMemoryFree(pOutput); + } + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and need persist", vid, + pRSmaInfo->suid, i + 1, len); +#if 1 + if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) { + smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid, + i + 1, terrstr(terrno)); + } else { + smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1); + } +#endif + } + + if (!isFileCreated) { + char qTaskInfoFName[TSDB_FILENAME_LEN]; + tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdInitTFile(&tFile, pTfs, qTaskInfoFName); + tdCreateTFile(&tFile, pTfs, true, -1); + + isFileCreated = true; + } + len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid)); + tdAppendTFile(&tFile, &len, sizeof(len), &toffset); + tdAppendTFile(&tFile, &type, sizeof(type), &toffset); + tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset); + tdAppendTFile(&tFile, pOutput, len, &toffset); + + taosMemoryFree(pOutput); + } infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); } -_end: +_normal: + if (isFileCreated) { + if (tdUpdateTFileHeader(&tFile) < 0) { + smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); + tdCloseTFile(&tFile); + tdRemoveTFile(&tFile); + goto _err; + } else { + smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_FILE_FULL_NAME(&tFile)); + } - if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); tdCloseTFile(&tFile); - tdRemoveTFile(&tFile); - return NULL; + + char newFName[TSDB_FILENAME_LEN]; + strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); + char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); + strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); + if (taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName) != 0) { + smaError("vgId:%d, failed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + goto _err; + } else { + smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + } } - - tdCloseTFile(&tFile); - - char newFName[TSDB_FILENAME_LEN]; - strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); - char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); - strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); - taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName); - - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); - return NULL; + goto _end; _err: - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); - // remove the .tmp file + if (isFileCreated) { + tdRemoveTFile(&tFile); + } +_end: + if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_INACTIVE, + TASK_TRIGGER_STAT_ACTIVE)) { + smaDebug("vgId:%d, persist task is active again", vid); + } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug("vgId:%d, persist task is cancelled", vid); + } else { + smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + ASSERT(0); + } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + taosThreadExit(NULL); return NULL; } static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { - smaWarn("%s:%d entry ", __func__, __LINE__); - TdThread threadId; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + TdThread tid; - if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { - smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno)); + if (taosThreadCreate(&tid, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { + if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_INACTIVE, + TASK_TRIGGER_STAT_ACTIVE)) { + smaDebug("persist task is active again"); + } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug(" persist task is cancelled and set finished"); + } else { + smaWarn("persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + ASSERT(0); + } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); } taosThreadAttrDestroy(&thAttr); - smaWarn("%s:%d end ", __func__, __LINE__); } /** @@ -696,17 +864,283 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *pRSmaStat = param; + int8_t tmrStat = + atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + switch (tmrStat) { + case TASK_TRIGGER_STAT_ACTIVE: { + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 1); + if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug("rsma persistence start since active"); - if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) { - smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId()); - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE); + // start persist task + tdRSmaPersistTask(pRSmaStat); - // execution - tdRSmaPersistTask(pRSmaStat); - } else { - smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__, - taosGetSelfPthreadId()); + taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, + &pRSmaStat->tmrId); + } else { + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + } + } break; + case TASK_TRIGGER_STAT_CANCELLED: { + atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); + smaDebug("rsma persistence not start since cancelled and finished"); + } break; + case TASK_TRIGGER_STAT_INACTIVE: { + smaDebug("rsma persistence not start since inactive"); + } break; + case TASK_TRIGGER_STAT_INIT: { + smaDebug("rsma persistence not start since init"); + } break; + default: { + smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); + ASSERT(0); + } break; + } +} + +int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + + // step 1: iterate all stables to restore the rsma env + + SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); + if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { + smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr()); + return TSDB_CODE_FAILED; } - taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); -} \ No newline at end of file + if (taosArrayGetSize(suidList) == 0) { + smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode)); + return TSDB_CODE_SUCCESS; + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) { + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + smaDebug("suid [%d] is %" PRIi64, i, suid); + if (metaGetTableEntryByUid(&mr, suid) < 0) { + smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + SRSmaParam *param = &mr.me.stbEntry.rsmaParam; + for (int i = 0; i < 2; ++i) { + smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode), + suid, i, param->maxdelay[i], i, param->watermark[i]); + } + if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { + smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + } + } + + // step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore + STFile tFile = {0}; + char qTaskInfoFName[TSDB_FILENAME_LEN]; + + tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName); + if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { + goto _err; + } + if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { + goto _err; + } + SRSmaQTaskFIter fIter = {0}; + if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { + goto _err; + } + SRSmaQTaskInfoItem infoItem = {0}; + bool isEnd = false; + int32_t code = 0; + while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) { + if (isEnd) { + break; + } + if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) { + break; + } + } + tdRSmaQTaskInfoIterDestroy(&fIter); + + if (code < 0) { + goto _err; + } + + metaReaderClear(&mr); + taosArrayDestroy(suidList); + return TSDB_CODE_SUCCESS; +_err: + ASSERT(0); + metaReaderClear(&mr); + taosArrayDestroy(suidList); + smaError("failed to restore rsma info since %s", terrstr()); + return TSDB_CODE_FAILED; +} + +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); + SRSmaInfo *pRSmaInfo = NULL; + void *qTaskInfo = NULL; + + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (infoItem->type == 1) { + qTaskInfo = pRSmaInfo->items[0].taskInfo; + } else if (infoItem->type == 2) { + qTaskInfo = pRSmaInfo->items[1].taskInfo; + } else { + ASSERT(0); + } + + if (!qTaskInfo) { + smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) { + smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid, + infoItem->type, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid, + infoItem->type); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) { + memset(pIter, 0, sizeof(*pIter)); + pIter->pTFile = pTFile; + pIter->offset = TD_FILE_HEAD_SIZE; + + if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { + return TSDB_CODE_FAILED; + } + + if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { + pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; + } else { + pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; + } + + if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { + pIter->nAlloc = TD_FILE_HEAD_SIZE; + } + + pIter->buf = taosMemoryMalloc(pIter->nAlloc); + if (!pIter->buf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) { + STFile *pTFile = pIter->pTFile; + int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; + + if (pIter->offset >= pIter->fsize) { + *isFinish = true; + return TSDB_CODE_SUCCESS; + } + + if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { + nBytes = pIter->fsize - pIter->offset; + } + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + int32_t infoLen = 0; + taosDecodeFixedI32(pIter->buf, &infoLen); + if (infoLen > nBytes) { + ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE); + pIter->nAlloc = infoLen; + void *pBuf = taosMemoryRealloc(pIter->buf, infoLen); + if (!pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->buf = pBuf; + nBytes = infoLen; + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + } + + pIter->offset += nBytes; + pIter->nBytes = nBytes; + pIter->nBufPos = 0; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) { + while (1) { + // block iter + bool isFinish = false; + if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + if (isFinish) { + *isEnd = true; + return TSDB_CODE_SUCCESS; + } + + // consume the block + int32_t qTaskInfoLenWithHead = 0; + pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); + if (qTaskInfoLenWithHead < 0) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FAILED; + } + while (1) { + if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { + pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type); + pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid); + pItem->qTaskInfo = pIter->buf; + pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); + // do the restore job + printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset); + + pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len); + pIter->nBufPos += qTaskInfoLenWithHead; + + pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); + continue; + } + // prepare and load next block in the file + pIter->offset -= (pIter->nBytes - pIter->nBufPos); + break; + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 76690a3bde..1f60da0b0a 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -15,7 +15,7 @@ #include "sma.h" -#define TD_FILE_HEAD_SIZE 512 +// smaFileUtil ================ #define TD_FILE_STATE_OK 0 #define TD_FILE_STATE_BAD 1 @@ -32,7 +32,7 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->ftype); tlen += taosEncodeFixedU32(buf, pInfo->fver); - tlen += taosEncodeFixedU64(buf, pInfo->fsize); + tlen += taosEncodeFixedI64(buf, pInfo->fsize); return tlen; } @@ -41,7 +41,7 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); buf = taosDecodeFixedU32(buf, &(pInfo->fver)); - buf = taosDecodeFixedU64(buf, &(pInfo->fsize)); + buf = taosDecodeFixedI64(buf, &(pInfo->fsize)); return buf; } @@ -69,6 +69,11 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { return loffset; } +int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) { + ASSERT(TD_FILE_OPENED(pTFile)); + return taosFStatFile(pTFile->pFile, size, NULL); +} + int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { ASSERT(TD_FILE_OPENED(pTFile)); @@ -235,4 +240,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp return 0; } -int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } \ No newline at end of file +int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } + +// smaXXXUtil ================ +// ... \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ab31d65d68..03f18cc766 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2876,6 +2876,33 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) { return TSDB_CODE_SUCCESS; } +/** + * @brief Get all suids since suid + * + * @param pMeta + * @param suid return all suids in one vnode if suid is 0 + * @param list + * @return int32_t + */ +int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { + SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid); + if(!pCur) { + return TSDB_CODE_FAILED; + } + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + static void destroyHelper(void* param) { if (param == NULL) { return; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3715866bb8..dbbe3d3750 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -72,7 +72,12 @@ int vnodeBegin(SVnode *pVnode) { return 0; } -int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; } +int vnodeShouldCommit(SVnode *pVnode) { + if (pVnode->inUse) { + return pVnode->inUse->size > pVnode->config.szBuf / 3; + } + return false; +} int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { char fname[TSDB_FILENAME_LEN]; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index dfc258b42b..4d73dbc406 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -152,12 +152,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { return pVnode; _err: + if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - if (pVnode->pSma) smaClose(pVnode->pSma); + tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); @@ -166,13 +167,14 @@ _err: void vnodeClose(SVnode *pVnode) { if (pVnode) { + smaCloseEnv(pVnode->pSma); vnodeCommit(pVnode); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); tqClose(pVnode->pTq); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); - smaClose(pVnode->pSma); + smaCloseEx(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 4c56180979..b998f40195 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -137,6 +137,25 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr()); } else { code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType)); + if (code == 1) { + do { + static int32_t cnt = 0; + if (cnt++ % 1000 == 1) { + vInfo("vgId:%d, msg:%p apply right now, apply index:%ld, msgtype:%s,%d", vgId, pMsg, + pMsg->info.conn.applyIndex, TMSG_INFO(pMsg->msgType), pMsg->msgType); + } + } while (0); + + SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; + if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { + rsp.code = terrno; + vInfo("vgId:%d, msg:%p failed to apply right now since %s", vgId, pMsg, terrstr()); + } + + if (rsp.info.handle != NULL) { + tmsgSendRsp(&rsp); + } + } } } @@ -163,10 +182,12 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; tmsgSendRedirectRsp(&rsp, &newEpSet); } else { - if (terrno != 0) code = terrno; - vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code); - SRpcMsg rsp = {.code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); + if (code != 1) { + if (terrno != 0) code = terrno; + vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code); + SRpcMsg rsp = {.code = code, .info = pMsg->info}; + tmsgSendRsp(&rsp); + } } vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); @@ -260,7 +281,7 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { @@ -359,34 +380,18 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c SyncIndex beginIndex = SYNC_INDEX_INVALID; char logBuf[256] = {0}; - if (pFsm->FpGetSnapshotInfo != NULL) { - (*pFsm->FpGetSnapshotInfo)(pFsm, &snapshot); - beginIndex = snapshot.lastApplyIndex; - } + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + beginIndex); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - if (cbMeta.index > beginIndex) { - snprintf( - logBuf, sizeof(logBuf), - "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); - syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); - rpcMsg.info.conn.applyIndex = cbMeta.index; - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); - - } else { - char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), - "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, " - "beginIndex :%ld\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - beginIndex); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - } + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + rpcMsg.info.conn.applyIndex = cbMeta.index; + tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 36f81e86ff..1cc3f9b874 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -403,7 +403,7 @@ typedef struct SIntervalAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter - + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. @@ -730,15 +730,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock, - SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, - SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream); + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, @@ -808,9 +805,10 @@ int32_t getMaximumIdleDurationSec(); * ops: root operator * data: *data save the result of encode, need to be freed by caller * length: *length save the length of *data + * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ -int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length); +int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal); /* * ops: root operator, created by caller diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 374a3a736d..be6ca5145a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -159,7 +159,12 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { } SArray* createSortInfo(SNodeList* pNodeList) { - size_t numOfCols = LIST_LENGTH(pNodeList); + size_t numOfCols = 0; + if (pNodeList != NULL) { + numOfCols = LIST_LENGTH(pNodeList); + } else { + numOfCols = 0; + } SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index a9e1e03178..e4c0959185 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -222,7 +222,13 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { return TSDB_CODE_INVALID_PARA; } - return encodeOperator(pTaskInfo->pRoot, pOutput, len); + int32_t nOptrWithVal = 0; + int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + taosMemoryFreeClear(*pOutput); + *len = 0; + } + return code; } int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f18da72f58..ad0d5c1447 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -273,8 +273,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR } // 1. close current opened time window - if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && - pResult->offset != pResultRowInfo->cur.offset))) { + if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) { SResultRowPosition pos = pResultRowInfo->cur; SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); releaseBufPage(pResultBuf, pPage); @@ -2929,6 +2928,13 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len int32_t totalSize = sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); + // no result + if (getTotalBufSize(pSup->pResultBuf) == 0) { + *result = NULL; + *length = 0; + return TSDB_CODE_SUCCESS; + } + *result = (char*)taosMemoryCalloc(1, totalSize); if (*result == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -4217,8 +4223,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); - pOptr = - createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, + pTaskInfo, isStream); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; @@ -4262,17 +4268,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; - - SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); - int32_t numOfOutputCols = 0; - SArray* pColList = - extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); - pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo); + pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -4465,12 +4461,12 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa return 0; } -int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { +int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) { int32_t code = TDB_CODE_SUCCESS; char* pCurrent = NULL; int32_t currLength = 0; if (ops->fpSet.encodeResultRow) { - if (result == NULL || length == NULL) { + if (result == NULL || length == NULL || nOptrWithVal == NULL) { return TSDB_CODE_TSC_INVALID_INPUT; } code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength); @@ -4481,8 +4477,15 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { *result = NULL; } return code; + } else if (currLength == 0) { + ASSERT(!pCurrent); + goto _downstream; } + ++(*nOptrWithVal); + + ASSERT(currLength >= 0); + if (*result == NULL) { *result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t)); if (*result == NULL) { @@ -4508,8 +4511,9 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { *length = *(int32_t*)(*result); } +_downstream: for (int32_t i = 0; i < ops->numOfDownstream; ++i) { - code = encodeOperator(ops->pDownstream[i], result, length); + code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal); if (code != TDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 47a195dc85..c51ef44154 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -448,7 +448,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } // do prepare for the next round table scan operation tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); @@ -467,7 +467,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } while (pTableScanInfo->scanTimes < total) { @@ -492,7 +492,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); pTableScanInfo->curTWinIdx = 0; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 96549ab6ae..44ff4c1c90 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -142,7 +142,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - SSDataBlock *pBlock = pOperator->fpSet.getNextFn(pOperator); + SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); return pBlock; } @@ -443,7 +443,7 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) { SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo) { SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { goto _error; } @@ -475,8 +475,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL, - NULL, getGroupSortExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + NULL, NULL, getGroupSortExplainExecInfo); int32_t code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -494,7 +494,7 @@ _error: //===================================================================================== // Multiway Sort Merge operator -typedef struct SMultiwaySortMergeOperatorInfo { +typedef struct SMultiwayMergeOperatorInfo { SOptrBasicInfo binfo; int32_t bufPageSize; @@ -504,16 +504,17 @@ typedef struct SMultiwaySortMergeOperatorInfo { SSortHandle* pSortHandle; SArray* pColMatchInfo; // for index map from table scan output - SSDataBlock* pInputBlock; - int64_t startTs; // sort start time - bool hasGroupId; - uint64_t groupId; - STupleHandle *prefetchedTuple; -} SMultiwaySortMergeOperatorInfo; + SSDataBlock* pInputBlock; + int64_t startTs; // sort start time + bool groupSort; + bool hasGroupId; + uint64_t groupId; + STupleHandle* prefetchedTuple; +} SMultiwayMergeOperatorInfo; -int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -527,8 +528,9 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { pInfo->pInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - tsortSetCompareGroupId(pInfo->pSortHandle, true); - + + tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; @@ -550,8 +552,8 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; blockDataCleanup(pDataBlock); @@ -564,35 +566,46 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData while (1) { STupleHandle* pTupleHandle = NULL; - if (pInfo->prefetchedTuple == NULL) { + if (pInfo->groupSort) { + if (pInfo->prefetchedTuple == NULL) { + pTupleHandle = tsortNextTuple(pHandle); + } else { + pTupleHandle = pInfo->prefetchedTuple; + pInfo->groupId = tsortGetGroupId(pTupleHandle); + pInfo->prefetchedTuple = NULL; + } + } + else { pTupleHandle = tsortNextTuple(pHandle); - } else { - pTupleHandle = pInfo->prefetchedTuple; - pInfo->groupId = tsortGetGroupId(pTupleHandle); - pInfo->prefetchedTuple = NULL; + pInfo->groupId = 0; } if (pTupleHandle == NULL) { break; } - uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); - if (!pInfo->hasGroupId) { - pInfo->groupId = tupleGroupId; - pInfo->hasGroupId = true; - appendOneRowToDataBlock(p, pTupleHandle); - } else if (pInfo->groupId == tupleGroupId) { - appendOneRowToDataBlock(p, pTupleHandle); + if (pInfo->groupSort) + { + uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); + if (!pInfo->hasGroupId) { + pInfo->groupId = tupleGroupId; + pInfo->hasGroupId = true; + appendOneRowToDataBlock(p, pTupleHandle); + } else if (pInfo->groupId == tupleGroupId) { + appendOneRowToDataBlock(p, pTupleHandle); + } else { + pInfo->prefetchedTuple = pTupleHandle; + break; + } } else { - pInfo->prefetchedTuple = pTupleHandle; - break; + appendOneRowToDataBlock(p, pTupleHandle); } if (p->info.rows >= capacity) { break; } } - if (p->info.rows > 0) {// todo extract method + if (p->info.rows > 0) { // todo extract method blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { @@ -614,13 +627,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } -SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { +SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; int32_t code = pOperator->fpSet._openFn(pOperator); if (code != TSDB_CODE_SUCCESS) { @@ -637,8 +650,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { return pBlock; } -void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param; +void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); @@ -646,11 +659,11 @@ void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pColMatchInfo); } -int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { +int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; @@ -658,24 +671,35 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx return TSDB_CODE_SUCCESS; } -SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, - SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo, - SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { - SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t rowSize = pResBlock->info.rowSize; +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numStreams, + SMergePhysiNode* pMergePhyNode, SExecTaskInfo* pTaskInfo) { + SPhysiNode* pPhyNode = (SPhysiNode*)pMergePhyNode; + + SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + int32_t rowSize = pResBlock->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { goto _error; } + SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + int32_t numOfOutputCols = 0; + SArray* pColMatchColInfo = + extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); + SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); initResultSizeInfo(pOperator, 1024); + pInfo->groupSort = pMergePhyNode->groupSort; pInfo->binfo.pRes = pResBlock; pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pInputBlock = pInputBlock; - pOperator->name = "MultiwaySortMerge"; + pOperator->name = "MultiwayMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; @@ -687,9 +711,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, // one additional is reserved for merged result. pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); - pOperator->fpSet = - createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, - destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo); int32_t code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b7d39207d8..963e714972 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -969,6 +969,12 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &pInfo->order, &scanFlag); + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup =& pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, + pExprSup->numOfExprs, NULL); + } + // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); @@ -1381,6 +1387,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again @@ -1498,18 +1509,27 @@ void increaseTs(SqlFunctionCtx* pCtx) { SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) { + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; + + if (pPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->primaryTsIndex = primaryTsSlotId; @@ -2473,7 +2493,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; + SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 0820e884ce..f7e22cb151 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -191,6 +191,11 @@ bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool uniqueFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t uniqueFunction(SqlFunctionCtx *pCtx); +bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool modeFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t modeFunction(SqlFunctionCtx *pCtx); +int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t twaFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 168df674d0..fbebb12cc3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1045,20 +1045,29 @@ static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int3 return translateFirstLastImpl(pFunc, pErrBuf, len, false); } -static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { +static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isUnique) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (!nodesExprHasColumn(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE must contain columns"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns", + isUnique ? "UNIQUE" : "MODE"); } pFunc->node.resType = ((SExprNode*)pPara)->resType; return TSDB_CODE_SUCCESS; } +static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateUniqueMode(pFunc, pErrBuf, len, true); +} + +static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateUniqueMode(pFunc, pErrBuf, len, false); +} + static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); if (numOfParams == 0 || numOfParams > 2) { @@ -2117,6 +2126,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = uniqueFunction, .finalizeFunc = NULL }, + { + .name = "mode", + .type = FUNCTION_TYPE_MODE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateMode, + .getEnvFunc = getModeFuncEnv, + .initFunc = modeFunctionSetup, + .processFunc = modeFunction, + .finalizeFunc = modeFinalize, + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c4d3a26ab4..06f24f39e7 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -32,6 +32,7 @@ #define TAIL_MAX_OFFSET 100 #define UNIQUE_MAX_RESULT_SIZE (1024 * 1024 * 10) +#define MODE_MAX_RESULT_SIZE UNIQUE_MAX_RESULT_SIZE #define HLL_BUCKET_BITS 14 // The bits of the bucket #define HLL_DATA_BITS (64 - HLL_BUCKET_BITS) @@ -246,6 +247,19 @@ typedef struct SUniqueInfo { char pItems[]; } SUniqueInfo; +typedef struct SModeItem { + int64_t count; + char data[]; +} SModeItem; + +typedef struct SModeInfo { + int32_t numOfPoints; + uint8_t colType; + int16_t colBytes; + SHashObj* pHash; + char pItems[]; +} SModeInfo; + typedef struct SDerivInfo { double prevValue; // previous value TSKEY prevTs; // previous timestamp @@ -4694,21 +4708,100 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) { return pInfo->numOfPoints; } -int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { +bool getModeFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SModeInfo) + MODE_MAX_RESULT_SIZE; + return true; +} + +bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (!functionSetup(pCtx, pResInfo)) { + return false; + } + + SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + pInfo->numOfPoints = 0; + pInfo->colType = pCtx->resDataInfo.type; + pInfo->colBytes = pCtx->resDataInfo.bytes; + if (pInfo->pHash != NULL) { + taosHashClear(pInfo->pHash); + } else { + pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + return true; +} + +static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) { + // ignore null elements + if (isNull) { + return; + } + + int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); + if (pHashItem == NULL) { + int32_t size = sizeof(SModeItem) + pInfo->colBytes; + SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size); + memcpy(pItem->data, data, pInfo->colBytes); + pItem->count += 1; + + taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); + pInfo->numOfPoints++; + } else { + (*pHashItem)->count += 1; + } +} + +int32_t modeFunction(SqlFunctionCtx* pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + int32_t startOffset = pCtx->offset; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + char* data = colDataGetData(pInputCol, i); + doModeAdd(pInfo, data, colDataIsNull_s(pInputCol, i)); + + if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { + taosHashCleanup(pInfo->pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + SET_VAL(pResInfo, 1, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + int32_t currentRow = pBlock->info.rows; - for (int32_t i = 0; i < pResInfo->numOfRes; ++i) { - SUniqueItem* pItem = (SUniqueItem*)(pInfo->pItems + i * (sizeof(SUniqueItem) + pInfo->colBytes)); - colDataAppend(pCol, i, pItem->data, false); - // TODO: handle ts output + int32_t resIndex; + int32_t maxCount = 0; + for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { + SModeItem* pItem = (SModeItem*)(pInfo->pItems + i * (sizeof(SModeItem) + pInfo->colBytes)); + if (pItem->count > maxCount) { + maxCount = pItem->count; + resIndex = i; + } else if (pItem->count == maxCount) { + resIndex = -1; + } } + SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); + colDataAppend(pCol, currentRow, pResItem->data, (resIndex == -1) ? true : false); + return pResInfo->numOfRes; } + bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(STwaInfo); return true; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 983cffe9dc..364ee0692f 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -110,7 +110,7 @@ static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); static int32_t udfdLoadUdf(char *udfName, SUdf *udf); -static bool udfdRpcRfp(int32_t code); +static bool udfdRpcRfp(int32_t code, tmsg_t msgType); static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); static int32_t udfdCloseClientRpc(); @@ -546,9 +546,12 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } return 0; } -static bool udfdRpcRfp(int32_t code) { +static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index 9443d5cb94..dfbae357ef 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -2,12 +2,8 @@ #include #include -#include "tudf.h" +#include "taosudf.h" -#undef malloc -#define malloc malloc -#undef free -#define free free DLL_EXPORT int32_t udf1_init() { return 0; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 1c270f5cf4..975832209e 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -1,13 +1,9 @@ #include #include #include +#include -#include "tudf.h" - -#undef malloc -#define malloc malloc -#undef free -#define free free +#include "taosudf.h" DLL_EXPORT int32_t udf2_init() { return 0; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index c4f4624355..202dd96581 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -337,7 +337,11 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { static bool isValidateTag(char* input) { if (!input) return false; for (size_t i = 0; i < strlen(input); ++i) { + #ifdef WINDOWS + if (input[i] < 0x20 || input[i] > 0x7E) return false; + #else if (isprint(input[i]) == 0) return false; + #endif } return true; } @@ -377,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMs char* jsonKey = item->string; if (!isValidateTag(jsonKey)) { + fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout); retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); goto end; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index eb87397f3b..1927afec0f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1616,6 +1616,82 @@ static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return rewriteUniqueOptimizeImpl(pCxt, pLogicSubplan, pIndef); } +// merge projects +static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pChild) || 1 < LIST_LENGTH(pChild->pChildren) || + NULL != pChild->pConditions || NULL != pNode->pLimit || NULL != pNode->pSlimit) { + return false; + } + return true; +} + +typedef struct SMergeProjectionsContext { + SProjectLogicNode* pChildProj; + int32_t errCode; +} SMergeProjectionsContext; + +static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { + SMergeProjectionsContext* pCxt = pContext; + SProjectLogicNode* pChildProj = pCxt->pChildProj; + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SNode* pTarget; + FOREACH(pTarget, ((SLogicNode*)pChildProj)->pTargets) { + if (nodesEqualNode(pTarget, *pNode)) { + SNode* pProjection; + FOREACH(pProjection, pChildProj->pProjections) { + if (0 == strcmp(((SColumnNode*)pTarget)->colName, ((SExprNode*)pProjection)->aliasName)) { + SNode* pExpr = nodesCloneNode(pProjection); + if (pExpr == NULL) { + pCxt->errCode = terrno; + return DEAL_RES_ERROR; + } + nodesDestroyNode(*pNode); + *pNode = pExpr; + } + } + } + } + return DEAL_RES_IGNORE_CHILD; + } + return DEAL_RES_CONTINUE; +} + +static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSelfNode) { + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSelfNode->pChildren, 0); + SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS}; + + nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt); + int32_t code = cxt.errCode; + + if (TSDB_CODE_SUCCESS == code) { + if (1 == LIST_LENGTH(pChild->pChildren)) { + SLogicNode* pGrandChild = (SLogicNode*)nodesListGetNode(pChild->pChildren, 0); + code = replaceLogicNode(pLogicSubplan, pChild, pGrandChild); + } else { // no grand child + NODES_CLEAR_LIST(pSelfNode->pChildren); + } + } + + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pChild->pChildren); + } + nodesDestroyNode((SNode*)pChild); + return code; +} + +static int32_t mergeProjectsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SLogicNode* pProjectNode = optFindPossibleNode(pLogicSubplan->pNode, mergeProjectsMayBeOptimized); + if (NULL == pProjectNode) { + return TSDB_CODE_SUCCESS; + } + + return mergeProjectsOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, @@ -1623,6 +1699,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}, diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 2e8e0e1227..beddec64c5 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -28,13 +28,13 @@ extern "C" { #include "trpc.h" #include "ttimer.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 1300 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 1000 +#define ELECT_TIMER_MS_MIN 1300 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 900 +#define HEARTBEAT_TIMER_MS 900 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 88d3065375..f3064e668d 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -50,7 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); - int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg, SyncIndex *pRetIndex); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 1bd0671fb5..d351dc50f4 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -169,7 +169,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); -int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak); +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak); // option bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); @@ -233,6 +233,7 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index); SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index); int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); +bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg); int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index efe8a65b77..521ca6068d 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -49,14 +49,14 @@ int32_t raftCfgClose(SRaftCfg *pRaftCfg); int32_t raftCfgPersist(SRaftCfg *pRaftCfg); int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex); -cJSON *syncCfg2Json(SSyncCfg *pSyncCfg); -char *syncCfg2Str(SSyncCfg *pSyncCfg); -char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg); +cJSON * syncCfg2Json(SSyncCfg *pSyncCfg); +char * syncCfg2Str(SSyncCfg *pSyncCfg); +char * syncCfg2SimpleStr(SSyncCfg *pSyncCfg); int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg); int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg); -cJSON *raftCfg2Json(SRaftCfg *pRaftCfg); -char *raftCfg2Str(SRaftCfg *pRaftCfg); +cJSON * raftCfg2Json(SRaftCfg *pRaftCfg); +char * raftCfg2Str(SRaftCfg *pRaftCfg); int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index b130186516..37f18a6388 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); void syncEntryLog(const SSyncRaftEntry* pObj); void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); +//----------------------------------- +typedef struct SRaftEntryCache { + SHashObj* pEntryHash; + int32_t maxCount; + int32_t currentCount; + TdThreadMutex mutex; + SSyncNode* pSyncNode; +} SRaftEntryCache; + +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount); +void raftCacheDestroy(SRaftEntryCache* pCache); +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry); +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index); +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheClear(struct SRaftEntryCache* pCache); + +cJSON* raftCache2Json(SRaftEntryCache* pObj); +char* raftCache2Str(SRaftEntryCache* pObj); +void raftCachePrint(SRaftEntryCache* pObj); +void raftCachePrint2(char* s, SRaftEntryCache* pObj); +void raftCacheLog(SRaftEntryCache* pObj); +void raftCacheLog2(char* s, SRaftEntryCache* pObj); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index f124867097..f3ed9e302b 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -29,9 +29,12 @@ extern "C" { #include "wal.h" typedef struct SSyncLogStoreData { - SSyncNode* pSyncNode; - SWal* pWal; + SSyncNode* pSyncNode; + SWal* pWal; + + TdThreadMutex mutex; SWalReadHandle* pWalHandle; + // SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index f00dfe6f50..14688f8912 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -35,12 +35,13 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 +//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; SSyncCfg lastConfig; @@ -56,36 +57,39 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader); -void snapshotSenderStop(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); -char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); +char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); + +//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { - bool start; - - int32_t ack; - void * pWriter; - SyncTerm term; - SyncTerm privateTerm; - SSnapshot snapshot; - - SSyncNode *pSyncNode; + bool start; + int32_t ack; + void *pWriter; + SyncTerm term; + SyncTerm privateTerm; + SSnapshot snapshot; SRaftId fromId; + SSyncNode *pSyncNode; } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); -bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); + +//--------------------------------------------------- +// on message int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index b7122b9c52..10f2745651 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -240,7 +240,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); ASSERT(pSender != NULL); - SSnapshot snapshot; + SSnapshot snapshot = {.data = NULL, + .lastApplyIndex = SYNC_INDEX_INVALID, + .lastApplyTerm = 0, + .lastConfigIndex = SYNC_INDEX_INVALID}; void* pReader = NULL; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot, NULL, &pReader); if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN && nextIndex <= snapshot.lastApplyIndex + 1 && @@ -249,10 +252,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries ASSERT(pReader != NULL); snapshotSenderStart(pSender, snapshot, pReader); - char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); - syncNodeEventLog(ths, eventLog); - taosMemoryFree(eventLog); - } else { // no snapshot if (pReader != NULL) { @@ -260,23 +259,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } } - /* - bool hasSnapshot = syncNodeHasSnapshot(ths); - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - - // start sending snapshot first time - // start here, stop by receiver - if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && - pMsg->privateTerm < pSender->privateTerm) { - snapshotSenderStart(pSender); - - char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); - syncNodeEventLog(ths, eventLog); - taosMemoryFree(eventLog); - } - */ - SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; // update nextIndex to sentryIndex @@ -300,5 +282,5 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrLog2("recv sync-append-entries-reply, after pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv sync-append-entries-reply, after pMatchIndex:", ths->pMatchIndex); - return ret; + return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index c92edae381..982d37826a 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -102,6 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } } + // maybe execute fsm if (newCommitIndex > pSyncNode->commitIndex) { SyncIndex beginIndex = pSyncNode->commitIndex + 1; SyncIndex endIndex = newCommitIndex; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 038c36c417..663745a7d7 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -30,7 +30,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -242,9 +242,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -281,7 +281,7 @@ static void *syncIOConsumerFunc(void *param) { if (io->FpOnSyncClientRequest != NULL) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); ASSERT(pSyncMsg != NULL); - io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg); + io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 5b432aeec4..10f0e0e335 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -126,7 +126,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3100d0525c..60f6ca7e7b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -50,7 +50,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths); // process message ---- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); // life cycle static void syncFreeNode(void* param); @@ -627,7 +627,7 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { return ret; } -int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; char eventLog[128]; @@ -664,13 +664,34 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { - ret = 0; + // optimized one replica + if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) { + SyncIndex retIndex; + int32_t code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, &retIndex); + if (code == 0) { + pMsg->info.conn.applyIndex = retIndex; + rpcFreeCont(rpcMsg.pCont); + syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + ret = 1; + sDebug("vgId:%d optimized index:%ld success, msgtype:%s,%d", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType), pMsg->msgType); + } else { + ret = -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("vgId:%d optimized index:%ld error, msgtype:%s,%d", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType), + pMsg->msgType); + } + } else { - ret = -1; - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("syncPropose pSyncNode->FpEqMsg is NULL"); + if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { + ret = 0; + } else { + ret = -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("enqueue msg error, FpEqMsg is NULL"); + } } + syncClientRequestDestroy(pSyncMsg); goto _END; @@ -1334,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + SyncIndex logLastIndex = SYNC_INDEX_INVALID; + SyncIndex logBeginIndex = SYNC_INDEX_INVALID; + if (pSyncNode->pLogStore != NULL) { + logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + } char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); char* printStr = ""; @@ -2377,7 +2403,7 @@ int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { // /\ UNCHANGED <> // -int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { int32_t ret = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); @@ -2436,6 +2462,14 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { rpcFreeCont(rpcMsg.pCont); } + if (pRetIndex != NULL) { + if (ret == 0 && pEntry != NULL) { + *pRetIndex = pEntry->index; + } else { + *pRetIndex = SYNC_INDEX_INVALID; + } + } + syncEntryDestory(pEntry); return ret; } @@ -2600,6 +2634,10 @@ static int32_t syncNodeProposeConfigChangeFinish(SSyncNode* ths, SyncReconfigFin return 0; } +bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { + return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1); +} + int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; @@ -2621,19 +2659,33 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, syncEntry2OriginalRpc(pEntry, &rpcMsg); // user commit - if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = ths->state; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = ths->pRaftStore->currentTerm; - cbMeta.flag = flag; + if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { + bool internalExecute = true; + if ((ths->replicaNum == 1) && ths->restoreFinish && (ths->vgId != 1)) { + internalExecute = false; + } - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "index:%ld, internalExecute:%d", i, internalExecute); + syncNodeEventLog(ths, logBuf); + } while (0); + + // execute fsm in apply thread, or execute outside syncPropose + if (internalExecute) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = flag; + + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } } // config change diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index f40eff57d9..9d16bed6c1 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { if (pSyncCfg != NULL) { int32_t len = 512; - char *s = taosMemoryMalloc(len); + char * s = taosMemoryMalloc(len); memset(s, 0, len); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); @@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -271,7 +271,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index ff334a76bb..225360630c 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } + +//----------------------------------- +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { + SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache)); + if (pCache == NULL) { + sError("vgId:%d raft cache create error", pSyncNode->vgId); + return NULL; + } + + pCache->pEntryHash = + taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pCache->pEntryHash == NULL) { + sError("vgId:%d raft cache create hash error", pSyncNode->vgId); + return NULL; + } + + taosThreadMutexInit(&(pCache->mutex), NULL); + pCache->maxCount = maxCount; + pCache->currentCount = 0; + pCache->pSyncNode = pSyncNode; + + return pCache; +} + +void raftCacheDestroy(SRaftEntryCache* pCache) { + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashCleanup(pCache->pEntryHash); + taosThreadMutexUnlock(&(pCache->mutex)); + taosThreadMutexDestroy(&(pCache->mutex)); + taosMemoryFree(pCache); + } +} + +// success, return 1 +// max count, return 0 +// error, return -1 +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) { + taosThreadMutexLock(&(pCache->mutex)); + + if (pCache->currentCount >= pCache->maxCount) { + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes); + ++(pCache->currentCount); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d", + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, + pEntry->index, pEntry->bytes); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = pEntry; + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheClear(struct SRaftEntryCache* pCache) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashClear(pCache->pEntryHash); + pCache->currentCount = 0; + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&(pCache->mutex)); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index cc606f4df8..d84c576714 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -257,6 +257,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, return -1; } + taosThreadMutexLock(&(pData->mutex)); + code = walReadWithHandle(pWalHandle, index); if (code != 0) { int32_t err = terrno; @@ -281,6 +283,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, terrno = saveErr; */ + taosThreadMutexUnlock(&(pData->mutex)); return code; } @@ -301,6 +304,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, terrno = saveErr; */ + taosThreadMutexUnlock(&(pData->mutex)); return code; } @@ -364,6 +368,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pData->pWal = pSyncNode->pWal; ASSERT(pData->pWal != NULL); + taosThreadMutexInit(&(pData->mutex), NULL); pData->pWalHandle = walOpenReadHandle(pData->pWal); ASSERT(pData->pWalHandle != NULL); @@ -408,9 +413,14 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { SSyncLogStoreData* pData = pLogStore->data; + + taosThreadMutexLock(&(pData->mutex)); if (pData->pWalHandle != NULL) { walCloseReadHandle(pData->pWalHandle); + pData->pWalHandle = NULL; } + taosThreadMutexUnlock(&(pData->mutex)); + taosThreadMutexDestroy(&(pData->mutex)); taosMemoryFree(pLogStore->data); taosMemoryFree(pLogStore); @@ -460,6 +470,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { + taosThreadMutexLock(&(pData->mutex)); + // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReadHandle* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -503,6 +515,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ + taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index a1ab95c00f..9f5cba6c66 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -216,7 +216,7 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { char *raftStore2Str(SRaftStore *pRaftStore) { cJSON *pJson = raftStore2Json(pRaftStore); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 48c1b70a04..d2cbabe226 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { } void syncRespMgrDestroy(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - taosHashCleanup(pObj->pRespHash); - taosThreadMutexUnlock(&(pObj->mutex)); - taosThreadMutexDestroy(&(pObj->mutex)); - taosMemoryFree(pObj); + if (pObj != NULL) { + taosThreadMutexLock(&(pObj->mutex)); + taosHashCleanup(pObj->pRespHash); + taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexDestroy(&(pObj->mutex)); + taosMemoryFree(pObj); + } } int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 4fc0128400..7b15be2d14 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -21,8 +21,10 @@ #include "syncUtil.h" #include "wal.h" +//---------------------------------- static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); +static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); //---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { @@ -49,7 +51,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; } else { - sError("snapshotSenderCreate cannot create sender"); + sError("vgId:%d, cannot create snapshot sender", pSyncNode->vgId); } return pSender; @@ -57,39 +59,59 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender != NULL) { + // free current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; } + + // close reader + if (pSender->pReader != NULL) { + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + pSender->pReader = NULL; + } + + // free sender taosMemoryFree(pSender); } } bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } -// begin send snapshot (current term, seq begin) +// begin send snapshot by snapshot, pReader void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) { ASSERT(!snapshotSenderIsStart(pSender)); - pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; - // init snapshot and reader ASSERT(pSender->pReader == NULL); pSender->pReader = pReader; pSender->snapshot = snapshot; + // init current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); } pSender->blockLen = 0; + // update term + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + ++(pSender->privateTerm); + + // update state + pSender->finish = false; + pSender->start = true; + pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + + // init last config if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { int32_t code = 0; SSyncRaftEntry *pEntry = NULL; + bool getLastConfig = false; + code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex, &pEntry); - - bool getLastConfig = false; if (code == 0) { ASSERT(pEntry != NULL); @@ -106,35 +128,35 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void syncEntryDestory(pEntry); } else { if (pSender->snapshot.lastConfigIndex == pSender->pSyncNode->pRaftCfg->lastConfigIndex) { - sTrace("vgId:%d sync sender get cfg from local", pSender->pSyncNode->vgId); + sTrace("vgId:%d, sync sender get cfg from local", pSender->pSyncNode->vgId); pSender->lastConfig = pSender->pSyncNode->pRaftCfg->cfg; getLastConfig = true; } } + // last config not found in wal, update to -1 if (!getLastConfig) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to -1", - pSender->snapshot.lastConfigIndex); - pSender->snapshot.lastConfigIndex = -1; - - char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - + SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex; + SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID; + pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to %ld", oldLastConfigIndex, + newLastConfigIndex); + char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } else { + // no last config memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); } - pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; - ++(pSender->privateTerm); - pSender->finish = false; - pSender->start = true; - // build begin msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; @@ -151,40 +173,47 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } -void snapshotSenderStop(SSyncSnapshotSender *pSender) { +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { + // close reader if (pSender->pReader != NULL) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); ASSERT(ret == 0); pSender->pReader = NULL; } + // free current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } + // update flag pSender->start = false; + pSender->finish = finish; - if (gRaftDetailLog) { - char *s = snapshotSender2Str(pSender); - sInfo("snapshotSenderStop %s", s); - taosMemoryFree(s); - } + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender stop"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } -// when sender receiver ack, call this function to send msg from seq +// when sender receive ack, call this function to send msg from seq // seq = ack + 1, already updated int32_t snapshotSend(SSyncSnapshotSender *pSender) { - // free memory last time (seq - 1) + // free memory last time (current seq - 1) if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); pSender->pCurrentBlock = NULL; @@ -198,7 +227,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->blockLen > 0) { // has read data } else { - // read finish + // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; } @@ -219,25 +248,28 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - - } else { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - } - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = NULL; + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { + eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish"); + } else { + eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending"); + } + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + return 0; } // send snapshot data from cache int32_t snapshotReSend(SSyncSnapshotSender *pSender) { - if (pSender->pCurrentBlock != NULL) { + // send current block data + if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { + // build msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; @@ -249,16 +281,20 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->seq = pSender->seq; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + // send msg SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } + return 0; } @@ -294,7 +330,6 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); @@ -324,7 +359,7 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { char *s = taosMemoryMalloc(len); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - char host[128]; + char host[64]; uint16_t port; syncUtilU642Addr(destId.addr, host, sizeof(host), &port); @@ -355,12 +390,12 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->term = pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = 0; pReceiver->snapshot.data = NULL; - pReceiver->snapshot.lastApplyIndex = -1; + pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; pReceiver->snapshot.lastApplyTerm = 0; - pReceiver->snapshot.lastConfigIndex = -1; + pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; } else { - sInfo("snapshotReceiverCreate cannot create receiver"); + sError("vgId:%d, cannot create snapshot receiver", pSyncNode->vgId); } return pReceiver; @@ -368,60 +403,54 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver != NULL) { + // close writer + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + } + + // free receiver taosMemoryFree(pReceiver); } } bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } -// begin receive snapshot msg (current term, seq begin) +// static do start by privateTerm, pBeginMsg +// receive first snapshot data +// write first block data static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { + // update state pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = privateTerm; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; pReceiver->fromId = pBeginMsg->srcId; + pReceiver->start = true; + // update snapshot pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex; pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm; pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex; + // write data ASSERT(pReceiver->pWriter == NULL); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); ASSERT(ret == 0); + + // event log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver start"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } -// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver -// if already start, force close, start again -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { - if (!snapshotReceiverIsStart(pReceiver)) { - // start - snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); - pReceiver->start = true; - - } else { - // already start - sInfo("snapshot recv, receiver already start"); - - // force close, abandon incomplete data - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); - ASSERT(ret == 0); - pReceiver->pWriter = NULL; - - // start again - snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); - pReceiver->start = true; - } - - if (gRaftDetailLog) { - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStart %s", s); - taosMemoryFree(s); - } -} - -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { +// force stop +static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { + // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); @@ -431,14 +460,106 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { pReceiver->start = false; - if (apply) { - // ++(pReceiver->privateTerm); + // event log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver force stop"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); +} + +// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver +// if already start, force close, start again +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { + if (!snapshotReceiverIsStart(pReceiver)) { + // first start + snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); + + } else { + // already start + sInfo("vgId:%d, snapshot recv, receiver already start", pReceiver->pSyncNode->vgId); + + // force close, abandon incomplete data + snapshotReceiverForceStop(pReceiver); + + // start again + snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); + } +} + +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; } - if (gRaftDetailLog) { - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStop %s", s); - taosMemoryFree(s); + pReceiver->start = false; + + // event log + do { + SSnapshot snapshot; + pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver stop"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); +} + +static void snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { + ASSERT(pMsg->seq == SYNC_SNAPSHOT_SEQ_END); + + if (pReceiver->pWriter != NULL) { + int32_t code = 0; + if (pMsg->dataLen > 0) { + code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, + pMsg->dataLen); + ASSERT(code == 0); + } + + code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true); + ASSERT(code == 0); + pReceiver->pWriter = NULL; + } + + pReceiver->ack = SYNC_SNAPSHOT_SEQ_END; + + // update commit index + if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) { + pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; + } + + // reset wal + pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex); + + // event log + do { + SSnapshot snapshot; + pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver got last data, finish, apply snapshot"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); +} + +static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { + ASSERT(pMsg->seq == pReceiver->ack + 1); + + if (pReceiver->pWriter != NULL) { + if (pMsg->dataLen > 0) { + int32_t code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, + pMsg->data, pMsg->dataLen); + ASSERT(code == 0); + } + pReceiver->ack = pMsg->seq; + + // event log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver receiving"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } @@ -520,33 +641,20 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // get receiver SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; bool needRsp = false; - int32_t writeCode = 0; // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - // begin + // begin, no data snapshotReceiverStart(pReceiver, pMsg->privateTerm, pMsg); - pReceiver->ack = pMsg->seq; needRsp = true; - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver begin"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { // end, finish FSM - writeCode = pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); - ASSERT(writeCode == 0); - - pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); - if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) { - pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; - } - - // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); - pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, pMsg->lastIndex); + snapshotReceiverFinish(pReceiver, pMsg); + snapshotReceiverStop(pReceiver); + needRsp = true; // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { @@ -561,89 +669,74 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { syncNodeDoConfigChange(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex); } - SSnapshot snapshot; - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver finish, apply snapshot"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - - pReceiver->pWriter = NULL; - snapshotReceiverStop(pReceiver, true); - pReceiver->ack = pMsg->seq; - needRsp = true; - - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver stop"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { - pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); - snapshotReceiverStop(pReceiver, false); + // force close + snapshotReceiverForceStop(pReceiver); needRsp = false; - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver force close"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // transfering if (pMsg->seq == pReceiver->ack + 1) { - writeCode = - pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); - ASSERT(writeCode == 0); - pReceiver->ack = pMsg->seq; + snapshotReceiverGotData(pReceiver, pMsg); } needRsp = true; - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver receiving"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } else { ASSERT(0); } + // send ack if (needRsp) { + // build msg SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; pRspMsg->term = pSyncNode->pRaftStore->currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->ack = pReceiver->ack; - pRspMsg->code = writeCode; - pRspMsg->privateTerm = pReceiver->privateTerm; + pRspMsg->ack = pReceiver->ack; // receiver maybe already closed + pRspMsg->code = 0; + pRspMsg->privateTerm = pReceiver->privateTerm; // receiver maybe already closed + // send msg SRpcMsg rpcMsg; syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); - syncSnapshotRspDestroy(pRspMsg); } + } else { + // error log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver term not equal"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } else { - syncNodeLog2("syncNodeOnSnapshotSendCb not follower", pSyncNode); + // error log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver not follower"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } return 0; } +static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + ASSERT(pMsg->ack == pSender->seq); + pSender->ack = pMsg->ack; + ++(pSender->seq); +} + // sender receives ack, set seq = ack + 1, send msg from seq // if ack == SYNC_SNAPSHOT_SEQ_END, stop sender int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - sInfo("recv SyncSnapshotRsp maybe replica already dropped"); - return 0; + sError("vgId:%d, recv sync-snapshot-rsp, maybe replica already dropped", pSyncNode->vgId); + return -1; } // get sender @@ -655,27 +748,49 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { // receiver ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - pSender->finish = true; - snapshotSenderStop(pSender); + snapshotSenderStop(pSender, true); return 0; } // send next msg if (pMsg->ack == pSender->seq) { // update sender ack - pSender->ack = pMsg->ack; - (pSender->seq)++; + snapshotSenderUpdateProgress(pSender, pMsg); snapshotSend(pSender); } else if (pMsg->ack == pSender->seq - 1) { snapshotReSend(pSender); } else { - ASSERT(0); + do { + char logBuf[64]; + snprintf(logBuf, sizeof(logBuf), "error ack, recv ack:%d, my seq:%d", pMsg->ack, pSender->seq); + char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + + return -1; } + } else { + // error log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender term not equal"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + + return -1; } } else { - syncNodeLog2("syncNodeOnSnapshotRspCb not leader", pSyncNode); + // error log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender not leader"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + + return -1; } return 0; diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index d6c2cbd34e..2c43312064 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -127,7 +127,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char *voteGranted2Str(SVotesGranted *pVotesGranted) { cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -256,7 +256,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { char *votesRespond2Str(SVotesRespond *pVotesRespond) { cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 27084286da..bc63462a9e 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "") add_executable(syncIndexMgrTest "") add_executable(syncLogStoreTest "") add_executable(syncEntryTest "") +add_executable(syncEntryCacheTest "") add_executable(syncRequestVoteTest "") add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") @@ -129,6 +130,10 @@ target_sources(syncEntryTest PRIVATE "syncEntryTest.cpp" ) +target_sources(syncEntryCacheTest + PRIVATE + "syncEntryCacheTest.cpp" +) target_sources(syncRequestVoteTest PRIVATE "syncRequestVoteTest.cpp" @@ -362,6 +367,11 @@ target_include_directories(syncEntryTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEntryCacheTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRequestVoteTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest sync gtest_main ) +target_link_libraries(syncEntryCacheTest + sync + gtest_main +) target_link_libraries(syncRequestVoteTest sync gtest_main diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp new file mode 100644 index 0000000000..787c08e507 --- /dev/null +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -0,0 +1,162 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncRaftEntry* createEntry(int i) { + int32_t dataLen = 20; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 88; + pEntry->originalRpcType = 99; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = i; + snprintf(pEntry->data, dataLen, "value%d", i); + + return pEntry; +} + +SSyncNode* createFakeNode() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + ASSERT(pSyncNode != NULL); + memset(pSyncNode, 0, sizeof(SSyncNode)); + + return pSyncNode; +} + +SRaftEntryCache* createCache(int maxCount) { + SSyncNode* pSyncNode = createFakeNode(); + ASSERT(pSyncNode != NULL); + + SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount); + ASSERT(pCache != NULL); + + return pCache; +} + +void test1() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test1 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + index = 3; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + raftCacheLog2((char*)"==test1 delete 1,3==", pCache); + + code = raftCacheClear(pCache); + ASSERT(code == 0); + raftCacheLog2((char*)"==clear all==", pCache); +} + +void test2() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test2 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + SSyncRaftEntry* pEntry; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + syncEntryLog2((char*)"==test2 get entry 1==", pEntry); + + index = 2; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry); + + // not found + index = 8; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry 8 not found=="); + + // not found + index = 9; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry pointer 9 not found=="); +} + +void test3() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + for (int i = 6; i < 10; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache); +} + +void test4() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test4 write 5 entries==", pCache); + + SyncIndex index; + index = 3; + SSyncRaftEntry* pEntry; + code = raftCacheGetAndDel(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry); + raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache); +} + +int main(int argc, char** argv) { + gRaftDetailLog = true; + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG; + + test1(); + test2(); + test3(); + test4(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index e3ceae0615..9e0c2ecc29 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -113,7 +113,7 @@ void test2() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; - //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); @@ -229,7 +229,7 @@ void test4() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); - //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { @@ -291,7 +291,7 @@ void test5() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { @@ -412,26 +412,23 @@ void test6() { do { SyncIndex firstVer = walGetFirstVer(pWal); SyncIndex lastVer = walGetLastVer(pWal); - bool isEmpty = walIsEmpty(pWal); + bool isEmpty = walIsEmpty(pWal); printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); } while (0); logStoreDestory(pLogStore); cleanup(); - - // restart init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; - do { SyncIndex firstVer = walGetFirstVer(pWal); SyncIndex lastVer = walGetLastVer(pWal); - bool isEmpty = walIsEmpty(pWal); + bool isEmpty = walIsEmpty(pWal); printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); } while (0); @@ -461,13 +458,13 @@ int main(int argc, char** argv) { } sTrace("gAssert : %d", gAssert); -/* - test1(); - test2(); - test3(); - test4(); - test5(); -*/ + /* + test1(); + test2(); + test3(); + test4(); + test5(); + */ test6(); return 0; diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index 302e29a091..ea1788c545 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -312,7 +312,7 @@ void test5() { pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - //pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5); for (int i = 6; i <= 10; ++i) { int32_t dataLen = 10; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 963a85922f..55db0b129a 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -229,7 +229,7 @@ typedef struct { SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); void transDestroyAsyncPool(SAsyncPool* pool); -int transSendAsync(SAsyncPool* pool, queue* mq); +int transAsyncSend(SAsyncPool* pool, queue* mq); #define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ do { \ diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c328629c4b..462debb247 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -52,7 +52,7 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - bool (*retry)(int32_t code); + bool (*retry)(int32_t code, tmsg_t msgType); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 4f7b19b539..cc2e95cfb3 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -164,11 +164,6 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { transSetDefaultAddr(thandle, ip, fqdn); } -// void rpcSetMsgTraceId(SRpcMsg* pMsg, STraceId uid) { -// SRpcHandleInfo* pInfo = &pMsg->info; -// pInfo->traceId = uid; -//} - int32_t rpcInit() { // impl later return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7374d1fffc..aba2e6957b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -967,7 +967,7 @@ void cliSendQuit(SCliThrd* thrd) { // cli can stop gracefully SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; - transSendAsync(thrd->asyncPool, &msg->q); + transAsyncSend(thrd->asyncPool, &msg->q); } void cliWalkCb(uv_handle_t* handle, void* arg) { if (!uv_is_closing(handle)) { @@ -1030,7 +1030,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { */ STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - if (pTransInst->retry != NULL && pTransInst->retry(code)) { + if (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { @@ -1138,7 +1138,7 @@ void transReleaseCliHandle(void* handle) { cmsg->msg = tmsg; cmsg->type = Release; - transSendAsync(pThrd->asyncPool, &cmsg->q); + transAsyncSend(pThrd->asyncPool, &cmsg->q); return; } @@ -1171,7 +1171,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra STraceId* trace = &pReq->info.traceId; tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); + ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); return; } @@ -1205,7 +1205,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - transSendAsync(pThrd->asyncPool, &(cliMsg->q)); + transAsyncSend(pThrd->asyncPool, &(cliMsg->q)); tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); @@ -1234,7 +1234,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid); - transSendAsync(thrd->asyncPool, &(cliMsg->q)); + transAsyncSend(thrd->asyncPool, &(cliMsg->q)); } } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index bff7d79bd3..fbe0951a46 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -202,7 +202,7 @@ void transDestroyAsyncPool(SAsyncPool* pool) { taosMemoryFree(pool->asyncs); taosMemoryFree(pool); } -int transSendAsync(SAsyncPool* pool, queue* q) { +int transAsyncSend(SAsyncPool* pool, queue* q) { int idx = pool->index; idx = idx % pool->nAsync; // no need mutex here diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 892d32696e..08363b3c7c 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -999,7 +999,7 @@ void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); - transSendAsync(pThrd->asyncPool, &msg->q); + transAsyncSend(pThrd->asyncPool, &msg->q); } void transCloseServer(void* arg) { @@ -1070,7 +1070,7 @@ void transReleaseSrvHandle(void* handle) { m->type = Release; tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); - transSendAsync(pThrd->asyncPool, &m->q); + transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(refMgt, refId); return; _return1: @@ -1099,7 +1099,7 @@ void transSendResponse(const STransMsg* msg) { STraceId* trace = (STraceId*)&msg->info.traceId; tGTrace("conn %p start to send resp (1/2)", exh->handle); - transSendAsync(pThrd->asyncPool, &m->q); + transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(refMgt, refId); return; _return1: @@ -1128,7 +1128,7 @@ void transRegisterMsg(const STransMsg* msg) { m->type = Register; tTrace("%s conn %p start to register brokenlink callback", transLabel(pThrd->pTransInst), exh->handle); - transSendAsync(pThrd->asyncPool, &m->q); + transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(refMgt, refId); return; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 0c6cd80f44..291d38bdca 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -157,14 +157,14 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { #ifdef WINDOWS bool code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (!code) { - printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); } return !code; #else int32_t code = rename(oldName, newName); if (code < 0) { - printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); } return code; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index ba07b6c3dd..9c66ac1df8 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -18,6 +18,10 @@ #include "os.h" #if defined(WINDOWS) +BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) { + printf("\n" TAOS_CONSOLE_PROMPT_HEADER); + return TRUE; +} #elif defined(_TD_DARWIN_64) #else #include @@ -124,7 +128,7 @@ int taosSetConsoleEcho(bool on) { void taosSetTerminalMode() { #if defined(WINDOWS) - // assert(0); + SetConsoleCtrlHandler(CtrlHandler, TRUE); #else struct termios newtio; @@ -158,7 +162,6 @@ void taosSetTerminalMode() { int32_t taosGetOldTerminalMode() { #if defined(WINDOWS) - // assert(0); #else /* Make sure stdin is a terminal. */ if (!isatty(STDIN_FILENO)) { @@ -176,7 +179,7 @@ int32_t taosGetOldTerminalMode() { void taosResetTerminalMode() { #if defined(WINDOWS) - // assert(0); + SetConsoleCtrlHandler(CtrlHandler, FALSE); #else if (tcsetattr(0, TCSANOW, &oldtio) != 0) { fprintf(stderr, "Fail to reset the terminal properties!\n"); diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 3f23cd8b5f..f80c2b217f 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -52,7 +52,7 @@ fi docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_TOOLS=true;make -j $THREAD_COUNT" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT" ret=$? exit $ret diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index b43f87be29..e9db4a5da3 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -637,5 +637,13 @@ class TDCom: column_value_str = ", ".join(str(v) for v in column_value_list) insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});' tsql.execute(insert_sql) - + def getOneRow(self, location, containElm): + res_list = list() + if 0 <= location < tdSql.queryRows: + for row in tdSql.queryResult: + if row[location] == containElm: + res_list.append(row) + return res_list + else: + tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}") tdCom = TDCom() diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index e5095c74dc..807398f70f 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -1,5 +1,76 @@ # -*- coding: utf-8 -*- +# basic type +TAOS_DATA_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", + "FLOAT", "DOUBLE", + "BOOL", + "BINARY", "NCHAR", "VARCHAR", + "TIMESTAMP", + # "MEDIUMBLOB", "BLOB", # add in 3.x + # "DECIMAL", "NUMERIC", # add in 3.x + "JSON", # only for tag +] + +TAOS_NUM_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", "FLOAT", "DOUBLE", + # "DECIMAL", "NUMERIC", # add in 3.x +] +TAOS_CHAR_TYPE = [ + "BINARY", "NCHAR", "VARCHAR", +] +TAOS_BOOL_TYPE = ["BOOL",] +TAOS_TS_TYPE = ["TIMESTAMP",] +TAOS_BIN_TYPE = [ + "MEDIUMBLOB", "BLOB", # add in 3.x +] + +TAOS_TIME_INIT = ["b", "u", "a", "s", "m", "h", "d", "w", "n", "y"] +TAOS_PRECISION = ["ms", "us", "ns"] +PRECISION_DEFAULT = "ms" +PRECISION = PRECISION_DEFAULT + +TAOS_KEYWORDS = [ + "ABORT", "CREATE", "IGNORE", "NULL", "STAR", + "ACCOUNT", "CTIME", "IMMEDIATE", "OF", "STATE", + "ACCOUNTS", "DATABASE", "IMPORT", "OFFSET", "STATEMENT", + "ADD", "DATABASES", "IN", "OR", "STATE_WINDOW", + "AFTER", "DAYS", "INITIALLY", "ORDER", "STORAGE", + "ALL", "DBS", "INSERT", "PARTITIONS", "STREAM", + "ALTER", "DEFERRED", "INSTEAD", "PASS", "STREAMS", + "AND", "DELIMITERS", "INT", "PLUS", "STRING", + "AS", "DESC", "INTEGER", "PPS", "SYNCDB", + "ASC", "DESCRIBE", "INTERVAL", "PRECISION", "TABLE", + "ATTACH", "DETACH", "INTO", "PREV", "TABLES", + "BEFORE", "DISTINCT", "IS", "PRIVILEGE", "TAG", + "BEGIN", "DIVIDE", "ISNULL", "QTIME", "TAGS", + "BETWEEN", "DNODE", "JOIN", "QUERIES", "TBNAME", + "BIGINT", "DNODES", "KEEP", "QUERY", "TIMES", + "BINARY", "DOT", "KEY", "QUORUM", "TIMESTAMP", + "BITAND", "DOUBLE", "KILL", "RAISE", "TINYINT", + "BITNOT", "DROP", "LE", "REM", "TOPIC", + "BITOR", "EACH", "LIKE", "REPLACE", "TOPICS", + "BLOCKS", "END", "LIMIT", "REPLICA", "TRIGGER", + "BOOL", "EQ", "LINEAR", "RESET", "TSERIES", + "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS", + "CACHE", "EXPLAIN", "LP", "ROW", "UNION", + "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED", + "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE", + "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS", + "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE", + "COLON", "FOR", "MINROWS", "SEMI", "USER", + "COLUMN", "FROM", "MINUS", "SESSION", "USERS", + "COMMA", "FSYNC", "MNODES", "SET", "USING", + "COMP", "GE", "MODIFY", "SHOW", "VALUES", + "COMPACT", "GLOB", "MODULES", "SLASH", "VARIABLE", + "CONCAT", "GRANTS", "NCHAR", "SLIDING", "VARIABLES", + "CONFLICT", "GROUP", "NE", "SLIMIT", "VGROUPS", + "CONNECTION", "GT", "NONE", "SMALLINT", "VIEW", + "CONNECTIONS", "HAVING", "NOT", "SOFFSET", "VNODES", + "CONNS", "ID", "NOTNULL", "STABLE", "WAL", + "COPY", "IF", "NOW", "STABLES", "WHERE", +] + # basic data type boundary TINYINT_MAX = 127 TINYINT_MIN = -128 @@ -11,7 +82,7 @@ SMALLINT_MAX = 32767 SMALLINT_MIN = -32768 SMALLINT_UN_MAX = 65535 -MALLINT_UN_MIN = 0 +SMALLINT_UN_MIN = 0 INT_MAX = 2147483647 INT_MIN = -2147483648 @@ -33,8 +104,8 @@ DOUBLE_MIN = -1.7E+308 # schema boundary BINARY_LENGTH_MAX = 16374 -NCAHR_LENGTH_MAX_ = 4093 -DBNAME_LENGTH_MAX_ = 64 +NCAHR_LENGTH_MAX = 4093 +DBNAME_LENGTH_MAX = 64 STBNAME_LENGTH_MAX = 192 STBNAME_LENGTH_MIN = 1 @@ -66,4 +137,32 @@ MNODE_SHM_SIZE_DEFAULT = 6292480 VNODE_SHM_SIZE_MAX = 2147483647 VNODE_SHM_SIZE_MIN = 6292480 -VNODE_SHM_SIZE_DEFAULT = 31458304 \ No newline at end of file +VNODE_SHM_SIZE_DEFAULT = 31458304 + +# time_init +TIME_MS = 1 +TIME_US = TIME_MS/1000 +TIME_NS = TIME_US/1000 + +TIME_S = 1000 * TIME_MS +TIME_M = 60 * TIME_S +TIME_H = 60 * TIME_M +TIME_D = 24 * TIME_H +TIME_W = 7 * TIME_D +TIME_N = 30 * TIME_D +TIME_Y = 365 * TIME_D + + +# session parameters +INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US + + +# streams and related agg-function +SMA_INDEX_FUNCTIONS = ["MIN", "MAX"] +ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"] +SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"] +WATERMARK_MAX = 900000 +WATERMARK_MIN = 0 + +MAX_DELAY_MAX = 900000 +MAX_DELAY_MIN = 1 \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 580fc8ee47..fe802dd9a3 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -21,6 +21,7 @@ import psutil import shutil import pandas as pd from util.log import * +from util.constant import * def _parse_datetime(timestr): try: @@ -117,8 +118,7 @@ class TDSql: col_name_list = [] col_type_list = [] self.cursor.execute(sql) - self.queryCols = self.cursor.description - for query_col in self.queryCols: + for query_col in self.cursor.description: col_name_list.append(query_col[0]) col_type_list.append(query_col[1]) except Exception as e: @@ -301,6 +301,41 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, elm, expect_elm) tdLog.exit("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) + def get_times(self, time_str, precision="ms"): + caller = inspect.getframeinfo(inspect.stack()[1][0]) + if time_str[-1] not in TAOS_TIME_INIT: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {time_str} not a standard taos time init") + if precision not in TAOS_PRECISION: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {precision} not a standard taos time precision") + + if time_str[-1] == TAOS_TIME_INIT[0]: + times = int(time_str[:-1]) * TIME_NS + if time_str[-1] == TAOS_TIME_INIT[1]: + times = int(time_str[:-1]) * TIME_US + if time_str[-1] == TAOS_TIME_INIT[2]: + times = int(time_str[:-1]) * TIME_MS + if time_str[-1] == TAOS_TIME_INIT[3]: + times = int(time_str[:-1]) * TIME_S + if time_str[-1] == TAOS_TIME_INIT[4]: + times = int(time_str[:-1]) * TIME_M + if time_str[-1] == TAOS_TIME_INIT[5]: + times = int(time_str[:-1]) * TIME_H + if time_str[-1] == TAOS_TIME_INIT[6]: + times = int(time_str[:-1]) * TIME_D + if time_str[-1] == TAOS_TIME_INIT[7]: + times = int(time_str[:-1]) * TIME_W + if time_str[-1] == TAOS_TIME_INIT[8]: + times = int(time_str[:-1]) * TIME_N + if time_str[-1] == TAOS_TIME_INIT[9]: + times = int(time_str[:-1]) * TIME_Y + + if precision == "ms": + return int(times) + elif precision == "us": + return int(times*1000) + elif precision == "ns": + return int(times*1000*1000) + def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b61bd1890..ab5117ced8 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -22,15 +22,19 @@ # ---- dnode ./test.sh -f tsim/dnode/balance_replica1.sim +#./test.sh -f tsim/dnode/balance_replica3.sim ./test.sh -f tsim/dnode/create_dnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim @@ -81,6 +85,7 @@ ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/distributeInterval0.sim +# ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim # ./test.sh -f tsim/stream/distributesession0.sim # ./test.sh -f tsim/stream/session0.sim ./test.sh -f tsim/stream/session1.sim diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat index 7a1a4bc7fa..1c698449c6 100644 --- a/tests/script/test-all.bat +++ b/tests/script/test-all.bat @@ -11,14 +11,17 @@ if not "%2" == "" ( ) for /F "usebackq tokens=*" %%i in (!caseFile!) do ( set line=%%i - if "!line:~,9!" == "./test.sh" ( - set /a a+=1 - echo !a! Processing %%i - call :GetTimeSeconds !time! - set time1=!_timeTemp! - echo Start at !time! - call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + call :CheckSkipCase %%i + if !skipCase! == false ( + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt || set /a errorlevel=8 + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) ) exit !exitNum! @@ -56,3 +59,10 @@ for %%a in (%tt%) do ( ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true ) +if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true ) +if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true ) +:goto eof \ No newline at end of file diff --git a/tests/script/tsim/dnode/balance_replica1.sim b/tests/script/tsim/dnode/balance_replica1.sim index 14f3f130fb..998d0654ab 100644 --- a/tests/script/tsim/dnode/balance_replica1.sim +++ b/tests/script/tsim/dnode/balance_replica1.sim @@ -117,7 +117,6 @@ if $rows != 6 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/balance_replica3.sim b/tests/script/tsim/dnode/balance_replica3.sim new file mode 100644 index 0000000000..b8996e259d --- /dev/null +++ b/tests/script/tsim/dnode/balance_replica3.sim @@ -0,0 +1,362 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 4 replica 3 + +print =============== step32 wait vgroup2 +$x = 0 +step32: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step32 +endi + +print =============== step33 wait vgroup3 +$x = 0 +step33: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step33 +endi + +print =============== step34 wait vgroup4 +$x = 0 +step34: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step34 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step35: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step35 +endi + +print =============== step36: create table +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql create table d1.c2 using st tags(1) +sql create table d1.c3 using st tags(1) +sql create table d1.c4 using st tags(1) +sql create table d1.c5 using st tags(1) +sql create table d1.c6 using st tags(1) +sql show d1.tables +if $rows != 6 then + return -1 +endi + +print =============== step4: start dnode5 +system sh/exec.sh -n dnode5 -s start +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != ready then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi +if $data(4)[4] != ready then + goto step4 +endi +if $data(5)[4] != ready then + goto step4 +endi + +print =============== step5: balance +sql balance vgroup + +print =============== step62 wait vgroup2 +$x = 0 +step62: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step62 +endi + +print =============== step63 wait vgroup3 +$x = 0 +step63: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step63 +endi + +print =============== step64 wait vgroup4 +$x = 0 +step64: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step64 +endi + +print =============== step65 wait vgroup5 +$x = 0 +step65: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step65 +endi + + + +print =============== step7: select data + +sql show d1.tables +print rows $rows +if $rows != 6 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim index 35a2466297..e1b7c4a3a4 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim @@ -28,7 +28,7 @@ step1: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -49,6 +49,128 @@ endi print =============== step3 create database sql create database d1 vgroups 4 replica 3 + +print =============== step32 wait vgroup2 +$x = 0 +step32: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step32 +endi + +print =============== step33 wait vgroup3 +$x = 0 +step33: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step33 +endi + +print =============== step34 wait vgroup4 +$x = 0 +step34: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step34 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step35: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step35 +endi + +print =============== step36: create table sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) @@ -57,21 +179,6 @@ if $rows != 1 then return -1 endi -sql show d1.vgroups -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -if $rows != 1 then - return -1 -endi -if $data(2)[3] != 2 then - return -1 -endi -if $data(2)[5] != 3 then - return -1 -endi -if $data(2)[7] != 4 then - return -1 -endi - print =============== step4: drop dnode 2 system sh/exec.sh -n dnode5 -s start $x = 0 @@ -85,7 +192,7 @@ step4: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -115,21 +222,133 @@ if $rows != 4 then return -1 endi -print show d1.vgroups +print =============== step62 wait vgroup2 +$x = 0 +step62: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi sql show d1.vgroups -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -if $rows != 1 then +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then return -1 endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step62 +endi +print =============== step63 wait vgroup3 +$x = 0 +step63: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step63 +endi -print =============== step6: select data +print =============== step64 wait vgroup4 +$x = 0 +step64: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step64 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step65: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step65 +endi + +print =============== step7: select data +print ===> $data00 $data01 sql show d1.tables if $rows != 1 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim index 3ea351f7a7..4855f4dccf 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim @@ -49,6 +49,50 @@ endi print =============== step3 create database sql create database d1 vgroups 1 replica 3 +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) @@ -72,13 +116,6 @@ if $data(2)[7] != 4 then return -1 endi -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT - - -return - - print =============== step4: drop dnode 2 system sh/exec.sh -n dnode5 -s start $x = 0 @@ -92,7 +129,7 @@ step4: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -116,8 +153,11 @@ sql drop dnode 2 print show dnodes; sql show dnodes; -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 if $rows != 4 then return -1 endi @@ -128,9 +168,9 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] if $rows != 1 then return -1 endi -#if $data(2)[3] != 3 then -# return -1 -#endi +if $data(2)[3] != 5 then + return -1 +endi print =============== step6: select data sql show d1.tables @@ -138,7 +178,37 @@ if $rows != 1 then return -1 endi -return +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step6 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim new file mode 100644 index 0000000000..d3b5a02a23 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim @@ -0,0 +1,178 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 1 + +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi + +print =============== step3: create database +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode 4 +sql redistribute vgroup 2 dnode 4 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: +print redistribute vgroup 2 dnode 4 +sql redistribute vgroup 2 dnode 4 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 9b11e16616..00c6e8a98d 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4 # vgroup not exist sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 # un changed -sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 +# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 # no enought vnodes sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4 # offline vnodes @@ -176,8 +176,6 @@ if $rows != 1 then return -1 endi -return - print =============== step32: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim new file mode 100644 index 0000000000..6614009497 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim @@ -0,0 +1,245 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 6 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 6 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi +if $data(6)[4] != ready then + goto step2 +endi + +print =============== step3: create database +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6 +sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: +print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim new file mode 100644 index 0000000000..c78d90e352 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim @@ -0,0 +1,263 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/deploy.sh -n dnode7 -i 7 +system sh/deploy.sh -n dnode8 -i 8 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode7 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode8 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 +sql create dnode $hostname port 7700 +sql create dnode $hostname port 7800 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 8 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +system sh/exec.sh -n dnode7 -s start +system sh/exec.sh -n dnode8 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 8 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi +if $data(6)[4] != ready then + goto step2 +endi +if $data(7)[4] != ready then + goto step2 +endi +if $data(8)[4] != ready then + goto step2 +endi + +print =============== step3: create database +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode 7 dnode 5 dnode 6 +sql redistribute vgroup 2 dnode 7 dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode 7 dnode 6 dnode 8 +sql redistribute vgroup 2 dnode 7 dnode 6 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + + +return + + +print =============== step44: +print redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode +sql redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 8 +sql redistribute vgroup 2dnode $follower1 dnode $follower2 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8 +sql redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 8936655974..a59e989d80 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -41,7 +41,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once watermark 1d into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim new file mode 100644 index 0000000000..cde5c7058f --- /dev/null +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -0,0 +1,291 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create dnode $hostname2 port 7200 + +system sh/exec.sh -n dnode2 -s start + +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); +sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts1 values(1648791223002,2,2,3,1.1); +sql insert into ts1 values(1648791233003,3,2,3,2.1); +sql insert into ts2 values(1648791243004,4,2,43,73.1); + +sql insert into ts1 values(1648791213002,24,22,23,4.1) (1648791243005,4,20,3,3.1); + +sleep 1000 + +sql insert into ts3 values(1648791213001,12,12,13,14.1) (1648791243005,14,30,30,30.1); + +$loop_count = 0 +loop1: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 5 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 14 then + print =====data02=$data02 + goto loop1 +endi + +# row 1 +if $data11 != 1 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop1 +endi + +#row2 +if $data21 != 1 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop1 +endi + +#row 3 +if $data31 != 3 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 22 then + print =====data32=$data32 + goto loop1 +endi + +print loop1 over + +sql insert into ts1 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1) (1648791243011,4,2,3,3.1)(1648791243012,34,32,33,3.1); + +$loop_count = 0 +loop2: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 6 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 18 then + print =====data02=$data02 + goto loop2 +endi + +# row 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +if $data12 != 6 then + print =====data12=$data12 + goto loop2 +endi + +#row2 +if $data21 != 2 then + print =====data21=$data21 + goto loop2 +endi + +if $data22 != 7 then + print =====data22=$data22 + goto loop2 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop2 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop2 +endi + +print loop2 over + +sql insert into ts4 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1); + +$loop_count = 0 +loop3: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 7 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 22 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data11 != 3 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 10 then + print =====data12=$data12 + goto loop3 +endi + +#row2 +if $data21 != 3 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 11 then + print =====data22=$data22 + goto loop3 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop3 +endi + +print loop3 over + +$loop_count = 0 +loop4: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 7 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 22 then + print =====data02=$data02 + goto loop4 +endi + +# row 1 +if $data11 != 3 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 10 then + print =====data12=$data12 + goto loop4 +endi + +#row2 +if $data21 != 3 then + print =====data21=$data21 + goto loop4 +endi + +if $data22 != 11 then + print =====data22=$data22 + goto loop4 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop4 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop4 +endi + +print loop4 over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/distributesession0.sim b/tests/script/tsim/stream/distributeSession0.sim similarity index 100% rename from tests/script/tsim/stream/distributesession0.sim rename to tests/script/tsim/stream/distributeSession0.sim diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 79daf01295..e3bbff9db5 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -56,8 +56,14 @@ echo charset UTF-8 >> %TAOS_CFG% set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" +set FILE_NAME=%FILE_NAME:/=\% + +start cmd /k "timeout /t 600 /NOBREAK && taskkill /f /im tsim.exe & exit /b" rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% +set result=false +%TSIM% -c %CFG_DIR% -f %FILE_NAME% && set result=true -%TSIM% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file +tasklist | grep timeout && taskkill /f /im timeout.exe +if "%result%" == "true" ( exit /b ) else ( exit /b 8 ) \ No newline at end of file diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index a4e40d1b0b..0f7a830634 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -16,273 +16,272 @@ import string from util.log import * from util.cases import * from util.sql import * +from util import constant +from util.common import * +from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } + self.tag_dict = { + 'ts_tag' : 'timestamp', + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': f'binary({self.binary_length})', + 't13': f'nchar({self.nchar_length})' + } + self.tag_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.tbnum = 1 + self.values_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.column_add_dict = { + 'col_time' : 'timestamp', + 'col_tinyint' : 'tinyint', + 'col_smallint' : 'smallint', + 'col_int' : 'int', + 'col_bigint' : 'bigint', + 'col_untinyint' : 'tinyint unsigned', + 'col_smallint' : 'smallint unsigned', + 'col_int' : 'int unsigned', + 'col_bigint' : 'bigint unsigned', + 'col_bool' : 'bool', + 'col_float' : 'float', + 'col_double' : 'double', + 'col_binary' : f'binary({constant.BINARY_LENGTH_MAX})', + 'col_nchar' : f'nchar({constant.NCAHR_LENGTH_MAX})' - def get_long_name(self, length, mode="mixed"): - """ - generate long name - mode could be numbers/letters/letters_mixed/mixed - """ - if mode == "numbers": - population = string.digits - elif mode == "letters": - population = string.ascii_letters.lower() - elif mode == "letters_mixed": - population = string.ascii_letters.upper() + string.ascii_letters.lower() - else: - population = string.ascii_letters.lower() + string.digits - return "".join(random.choices(population, k=length)) - - def alter_tb_tag_check(self): - tag_tinyint = random.randint(-127,129) - tag_int = random.randint(-2147483648,2147483647) - tag_smallint = random.randint(-32768,32768) - tag_bigint = random.randint(-2147483648,2147483647) - tag_untinyint = random.randint(0,256) - tag_unsmallint = random.randint(0,65536) - tag_unint = random.randint(0,4294967296) - tag_unbigint = random.randint(0,2147483647) - tag_binary = self.get_long_name(length=10, mode="letters") - tag_nchar = self.get_long_name(length=10, mode="letters") - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create stable if not exists {dbname}.{stbname} (col_ts timestamp, c1 int) tags (tag_ts timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20))') - tdSql.execute(f'create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags(now, 1, 2, 3, 4, 5, 6, 7, 8, 9.9, 10.1, True,"abc123","涛思数据")') - tdSql.execute(f'insert into {dbname}.{tbname} values(now, 1)') - tdSql.execute(f'alter table {dbname}.{tbname} set tag tag_ts = 1640966400000') - tdSql.execute(f'alter table {dbname}.{tbname} set tag `t1` = 11') - tdSql.query(f'select * from {dbname}.{stbname}') - tdSql.checkData(0,3,11) - tdSql.execute(f'alter table {dbname}.{tbname} set tag t1 = {tag_tinyint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t2 = {tag_smallint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t3 = {tag_int}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t4 = {tag_bigint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t5 = {tag_untinyint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t6 = {tag_unsmallint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t7 = {tag_unint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t8 = {tag_unbigint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t11 = false') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t12 = "{tag_binary}"') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t13 = "{tag_nchar}"') - tdSql.query(f'select * from {dbname}.{stbname}') - # bug TD-15899 - tdSql.checkData(0,2,'2022-01-01 00:00:00.000') - tdSql.checkData(0,3,tag_tinyint) - tdSql.checkData(0,4,tag_smallint) - tdSql.checkData(0,5,tag_int) - tdSql.checkData(0,6,tag_bigint) - tdSql.checkData(0,7,tag_untinyint) - tdSql.checkData(0,8,tag_unsmallint) - tdSql.checkData(0,9,tag_unint) - tdSql.checkData(0,10,tag_unbigint) + } + def alter_check_ntb(self): - tdSql.checkData(0,13,False) - tdSql.checkData(0,14,tag_binary) - tdSql.checkData(0,15,tag_nchar) + tdSql.prepare() + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in self.values_list: + tdSql.execute(f'insert into {self.ntbname} values({i})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter table {self.ntbname} add column {key} {values}') + tdSql.query(f'describe {self.ntbname}') + tdSql.checkRows(len(self.column_dict)+1) + tdSql.query(f'select {key} from {self.ntbname}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter table {self.ntbname} drop column {key}') + tdSql.query(f'describe {self.ntbname}') + tdSql.checkRows(len(self.column_dict)) + tdSql.error(f'select {key} from {self.ntbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.query(f'describe {self.ntbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.query(f'describe {self.ntbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter table {self.ntbname} modify column {key} {v}') + for key,values in self.column_dict.items(): + rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' + tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}') + tdSql.query(f'select {rename_str} from {self.ntbname}') + tdSql.checkRows(1) + + def alter_check_tb(self): + tag_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + tag_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + tag_int = random.randint(constant.INT_MIN,constant.INT_MAX) + tag_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + tag_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + tag_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + tag_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + tag_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + tag_bool = random.randint(0,100)%2 + tag_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + tag_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + tag_binary = tdCom.getLongName(self.binary_length) + tag_nchar = tdCom.getLongName(self.binary_length) + modify_column_dict = { + 'ts1' : 'timestamp', + 'c1': 'tinyint', + 'c2': 'smallint', + 'c3': 'int', + 'c4': 'bigint', + 'c5': 'tinyint unsigned', + 'c6': 'smallint unsigned', + 'c7': 'int unsigned', + 'c8': 'bigint unsigned', + 'c9': 'float', + 'c10': 'double', + 'c11': 'bool', + 'c12': f'binary({self.binary_length})', + 'c13': f'nchar({self.nchar_length})' + } + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for i in range(self.tbnum): + for k,v in modify_column_dict.items(): + tdSql.error(f'alter table {self.stbname}_{i} add column {k} {v}') + for k in self.column_dict.keys(): + tdSql.error(f'alter table {self.stbname}_{i} drop column {k}') + for k,v in self.column_dict.items(): + if 'binary' in v.lower(): + values = [f'binary({self.binary_length+1})', f'binary({self.binary_length-1})'] + for j in values: + tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}') + elif 'nchar' in v.lower(): + values = [f'nchar({self.nchar_length+1})', f'binary({self.nchar_length-1})'] + for j in values: + tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}') + else: + for values in self.column_dict.values(): + tdSql.error(f'alter table {self.stbname}_{i} modify column {k} {values}') + for k,v in self.tag_dict.items(): + if v.lower() == 'tinyint': + self.tag_check(i,k,tag_tinyint) + elif v.lower() == 'smallint': + self.tag_check(i,k,tag_smallint) + elif v.lower() == 'int': + self.tag_check(i,k,tag_int) + elif v.lower() == 'bigint': + self.tag_check(i,k,tag_bigint) + elif v.lower() == 'tinyint unsigned': + self.tag_check(i,k,tag_untinyint) + elif v.lower() == 'smallint unsigned': + self.tag_check(i,k,tag_unsmallint) + elif v.lower() == 'int unsigned': + self.tag_check(i,k,tag_unint) + elif v.lower() == 'bigint unsigned': + self.tag_check(i,k,tag_unbigint) + elif v.lower() == 'bool': + self.tag_check(i,k,tag_bool) + elif v.lower() == 'float': + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_float}') + tdSql.query(f'select {k} from {self.stbname}_{i}') + if abs(tdSql.queryResult[0][0] - tag_float)/tag_float<=0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + elif v.lower() == 'double': + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_double}') + tdSql.query(f'select {k} from {self.stbname}_{i}') + if abs(tdSql.queryResult[0][0] - tag_double)/tag_double<=0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + elif 'binary' in v.lower(): + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_binary}"') + tdSql.query(f'select {k} from {self.stbname}_{i}') + tdSql.checkData(0,0,tag_binary) + elif 'nchar' in v.lower(): + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_nchar}"') + tdSql.query(f'select {k} from {self.stbname}_{i}') + tdSql.checkData(0,0,tag_nchar) + + def tag_check(self,tb_no,tag,values): + tdSql.execute(f'alter table {self.stbname}_{tb_no} set tag {tag} = {values}') + tdSql.query(f'select {tag} from {self.stbname}_{tb_no}') + tdSql.checkData(0,0,values) + def alter_check_stb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter table {self.stbname} add column {key} {values}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + tdSql.query(f'select {key} from {self.stbname}_{i}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter table {self.stbname} drop column {key}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + tdSql.error(f'select {key} from {self.stbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter table {self.stbname} modify column {key} {v}') - # bug TD-16211 insert length more than setting binary and nchar - # error_tag_binary = self.get_long_name(length=21, mode="letters") - # error_tag_nchar = self.get_long_name(length=21, mode="letters") - # tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"') - # tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"') - error_tag_binary = self.get_long_name(length=25, mode="letters") - error_tag_nchar = self.get_long_name(length=25, mode="letters") - tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"') - tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"') - # bug TD-16210 modify binary to nchar - tdSql.error(f'alter table {dbname}.{tbname} modify tag t12 nchar(10)') - tdSql.execute(f"drop database {dbname}") - def alter_ntb_column_check(self): - ''' - alter ntb column check - ''' - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tbname = self.get_long_name(length=3, mode="letters") - tdLog.info('------------------normal table column check---------------------') - tdLog.info(f'-----------------create normal table {tbname}-------------------') - tdSql.execute(f'create table if not exists {dbname}.{tbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20))') - tdSql.execute(f'insert into {dbname}.{tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - # bug TD-15757 - tdSql.execute(f'alter table {dbname}.{tbname} add column c14 int') - tdSql.query(f'select c14 from {dbname}.{tbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {dbname}.{tbname} add column `c15` int') - tdSql.query(f'select c15 from {dbname}.{tbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {dbname}.{tbname} drop column c14') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter table {dbname}.{tbname} drop column `c15`') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(14) - #! TD-16422 - # tdSql.execute(f'alter table {dbname}.{tbname} add column c16 binary(10)') - # tdSql.query(f'describe {dbname}.{tbname}') - # tdSql.checkRows(15) - # tdSql.checkEqual(tdSql.queryResult[14][2],10) - # tdSql.execute(f'alter table {dbname}.{tbname} drop column c16') - - # tdSql.execute(f'alter table {dbname}.{tbname} add column c16 nchar(10)') - # tdSql.query(f'describe {dbname}.{tbname}') - # tdSql.checkRows(15) - # tdSql.checkEqual(tdSql.queryResult[14][2],10) - # tdSql.execute(f'alter table {dbname}.{tbname} drop column c16') - - - tdSql.execute(f'alter table {dbname}.{tbname} modify column c12 binary(30)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter table {dbname}.{tbname} modify column `c12` binary(35)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 binary(34)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 nchar(10)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 int') - tdSql.execute(f'alter table {dbname}.{tbname} modify column c13 nchar(30)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(13,2,30) - tdSql.execute(f'alter table {dbname}.{tbname} modify column `c13` nchar(35)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(13,2,35) - tdSql.error(f'alter table {dbname}.{tbname} modify column c13 nchar(34)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c13 binary(10)') - tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(1,0,'c21') - # !bug TD-16423 - # tdSql.error(f'select c1 from {dbname}.{tbname}') - # tdSql.query(f'select c21 from {dbname}.{tbname}') - # tdSql.checkData(0,1,1) - tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(1,0,'c1') - # !bug TD-16423 - # tdSql.error(f'select c1 from {dbname}.{tbname}') - # tdSql.query(f'select c1 from {dbname}.{tbname}') - # tdSql.checkData(0,1,1) - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int') - tdSql.error(f'alter table {dbname}.{tbname} modify column `c1` double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c9 double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c10 float') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)') - tdSql.execute(f'drop database {dbname}') - def alter_stb_column_check(self): - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create table {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'alter table {stbname} add column c14 int') - tdSql.query(f'select c14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {stbname} add column `c15` int') - tdSql.query(f'select c15 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter table {stbname} drop column c14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {stbname} drop column `c15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter table {stbname} modify column c12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter table {stbname} modify column `c12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter table {stbname} modify column `c12` binary(34)') - tdSql.execute(f'alter table {stbname} modify column c13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(13,2,30) - tdSql.error(f'alter table {stbname} modify column c13 nchar(29)') - tdSql.error(f'alter table {stbname} rename column c1 c21') - tdSql.error(f'alter table {stbname} modify column c1 int') - tdSql.error(f'alter table {stbname} modify column c4 int') - tdSql.error(f'alter table {stbname} modify column c8 int') - tdSql.error(f'alter table {stbname} modify column c1 unsigned int') - tdSql.error(f'alter table {stbname} modify column c9 double') - tdSql.error(f'alter table {stbname} modify column c10 float') - tdSql.error(f'alter table {stbname} modify column c11 int') - tdSql.execute(f'drop database {dbname}') - def alter_stb_tag_check(self): - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create table {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'insert into {tbname} values(now,1)') - - tdSql.execute(f'alter table {stbname} add tag t14 int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {stbname} add tag `t15` int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(18) - tdSql.execute(f'alter table {stbname} drop tag t14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter table {stbname} drop tag `t15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {stbname} modify tag t12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,30) - tdSql.execute(f'alter table {stbname} modify tag `t12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,35) - tdSql.error(f'alter table {stbname} modify tag `t12` binary(34)') - tdSql.execute(f'alter table {stbname} modify tag t13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(15,2,30) - tdSql.error(f'alter table {stbname} modify tag t13 nchar(29)') - tdSql.execute(f'alter table {stbname} rename tag t1 t21') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t21') - tdSql.execute(f'alter table {stbname} rename tag `t21` t1') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t1') - - for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: - for j in [1,2,3]: - tdSql.error(f'alter table {stbname} modify tag t{j} {i}') - for i in ['int','unsigned int','float','binary(10)','nchar(10)']: - tdSql.error(f'alter table {stbname} modify tag t8 {i}') - tdSql.error(f'alter table {stbname} modify tag t4 int') - tdSql.execute(f'drop database {dbname}') def run(self): - self.alter_tb_tag_check() - self.alter_ntb_column_check() - self.alter_stb_column_check() - self.alter_stb_tag_check() + self.alter_check_ntb() + self.alter_check_tb() + self.alter_check_stb() def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 5a0684e2ee..4b37eeb9a5 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -21,9 +21,9 @@ SINT_UN_COL = "c_sint_un" BINT_UN_COL = "c_bint_un" INT_UN_COL = "c_int_un" -BINARY_COL = "c8" -NCHAR_COL = "c9" -TS_COL = "c10" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] @@ -51,12 +51,28 @@ class DataSet: binary_data : List[str] = None nchar_data : List[str] = None + def __post_init__(self): + self.ts_data = [] + self.int_data = [] + self.bint_data = [] + self.sint_data = [] + self.tint_data = [] + self.int_un_data = [] + self.bint_un_data = [] + self.sint_un_data = [] + self.tint_un_data = [] + self.float_data = [] + self.double_data = [] + self.bool_data = [] + self.binary_data = [] + self.nchar_data = [] + class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) @property def create_databases_sql_err(self): @@ -87,28 +103,28 @@ class TDTestCase: @property def create_stable_sql_err(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ", f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m", - # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m", + # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s", # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " , - # watermark, maxdelay: [0, 900000], [ms, s, m, ?] - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u", + # watermark, max_delay: [0, 900000], [ms, s, m, ?] + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d", ] @@ -117,11 +133,11 @@ class TDTestCase: def create_stable_sql_current(self): return [ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m", - f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m", - f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m", - # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m", - # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m", + f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m", + f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m", + # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m", + # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m", ] def test_create_stb(self): @@ -135,7 +151,7 @@ class TDTestCase: tdSql.checkRows(len(self.create_stable_sql_current)) # tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database - # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0") + # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m") def test_create_databases(self): @@ -177,21 +193,6 @@ class TDTestCase: def __data_set(self, rows): data_set = DataSet() - # neg_data_set = DataSet() - data_set.ts_data = [] - data_set.int_data = [] - data_set.bint_data = [] - data_set.sint_data = [] - data_set.tint_data = [] - data_set.int_un_data = [] - data_set.bint_un_data = [] - data_set.sint_un_data = [] - data_set.tint_un_data = [] - data_set.float_data = [] - data_set.double_data = [] - data_set.bool_data = [] - data_set.binary_data = [] - data_set.nchar_data = [] for i in range(rows): data_set.ts_data.append(NOW + 1 * (rows - i)) @@ -226,6 +227,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -264,10 +266,10 @@ class TDTestCase: def run(self): self.rows = 10 - + tdSql.prepare() tdLog.printNoPrefix("==========step0:all check") - # self.all_test() + self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 5387970d56..d4434987a6 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -17,25 +17,34 @@ TINT_COL = "c_tint" FLOAT_COL = "c_float" DOUBLE_COL = "c_double" BOOL_COL = "c_bool" -TINT_UN_COL = "c_tint_un" -SINT_UN_COL = "c_sint_un" -BINT_UN_COL = "c_bint_un" -INT_UN_COL = "c_int_un" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" BINARY_COL = "c_binary" NCHAR_COL = "c_nchar" TS_COL = "c_ts" - - NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [BOOL_COL, ] TS_TYPE_COL = [TS_COL, ] +INT_TAG = "t_int" + +ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] +TAG_COL = [INT_TAG] + # insert data args： TIME_STEP = 10000 NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) +# init db/table +DBNAME = "db" +STBNAME = "stb1" +CTBNAME = "ct1" +NTBNAME = "nt1" + @dataclass class DataSet: @@ -73,19 +82,25 @@ class DataSet: @dataclass class SMAschema: - creation : str = "CREATE" - index_name : str = "sma_index_1" - index_flag : str = "SMA INDEX" - operator : str = "ON" - tbname : str = None - watermark : str = None - maxdelay : str = None - func : Tuple[str] = None - interval : Tuple[str] = None - sliding : str = None - other : Any = None - drop : str = "DROP" - drop_flag : str = "INDEX" + creation : str = "CREATE" + index_name : str = "sma_index_1" + index_flag : str = "SMA INDEX" + operator : str = "ON" + tbname : str = None + watermark : str = "5s" + max_delay : str = "6m" + func : Tuple[str] = None + interval : Tuple[str] = ("6m", "10s") + sliding : str = "6m" + other : Any = None + drop : str = "DROP" + drop_flag : str = "INDEX" + querySmaOptimize : int = 1 + show : str = "SHOW" + show_msg : str = "INDEXES" + show_oper : str = "FROM" + dbname : str = None + rollup_db : bool = False def __post_init__(self): if isinstance(self.other, dict): @@ -111,8 +126,8 @@ class SMAschema: self.watermark = v del self.other[k] - if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay: - self.maxdelay = v + if k.lower() == "max_delay" and isinstance(v, str) and not self.max_delay: + self.max_delay = v del self.other[k] if k.lower() == "functions" and isinstance(v, tuple) and not self.func: @@ -131,12 +146,36 @@ class SMAschema: self.drop_flag = v del self.other[k] + if k.lower() == "show_msg" and isinstance(v, str) and not self.show_msg: + self.show_msg = v + del self.other[k] + + if k.lower() == "dbname" and isinstance(v, str) and not self.dbname: + self.dbname = v + del self.other[k] + + if k.lower() == "show_oper" and isinstance(v, str) and not self.show_oper: + self.show_oper = v + del self.other[k] + + if k.lower() == "rollup_db" and isinstance(v, bool) and not self.rollup_db: + self.rollup_db = v + del self.other[k] + + + +# from ...pytest.util.sql import * +# from ...pytest.util.constant import * class TDTestCase: + updatecfgDict = {"querySmaOptimize": 1} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) + self.precision = "ms" + self.sma_count = 0 + self.sma_created_index = [] """ create sma index : @@ -155,13 +194,17 @@ class TDTestCase: if sma.func: sql += f" function({', '.join(sma.func)})" if sma.interval: - sql += f" interval({', '.join(sma.interval)})" + interval, offset = self.__get_interval_offset(sma.interval) + if offset: + sql += f" interval({interval}, {offset})" + else: + sql += f" interval({interval})" if sma.sliding: sql += f" sliding({sma.sliding})" if sma.watermark: sql += f" watermark {sma.watermark}" - if sma.maxdelay: - sql += f" maxdelay {sma.maxdelay}" + if sma.max_delay: + sql += f" max_delay {sma.max_delay}" if isinstance(sma.other, dict): for k,v in sma.other.items(): if isinstance(v,tuple) or isinstance(v, list): @@ -171,53 +214,305 @@ class TDTestCase: if isinstance(sma.other, tuple) or isinstance(sma.other, list): sql += " ".join(sma.other) if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str): - sql += sma.other + sql += f" {sma.other}" return sql - def sma_create_check(self, sma:SMAschema): + def __get_sma_func_col(self, func): + cols = [] + if isinstance(func, str): + cols.append( func.split("(")[-1].split(")")[0] ) + elif isinstance(func, tuple) or isinstance(func, list): + for func_col in func: + cols.append(func_col.split("(")[-1].split(")")[0]) + else: + cols = [] + return cols + + def __check_sma_func(self, func:tuple): + if not isinstance(func, str) and not isinstance(func, tuple) and not isinstance(func, list): + return False + if isinstance(func, str) : + if "(" not in func or ")" not in func: + return False + if func.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if func.split("(")[1].split(")")[0] not in ALL_COL and func.split("(")[1].split(")")[0] not in TAG_COL : + return False + if isinstance(func, tuple) or isinstance(func, list): + for arg in func: + if not isinstance(arg, str): + return False + if "(" not in arg or ")" not in arg: + return False + if arg.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if arg.split("(")[1].split(")")[0] not in ALL_COL and arg.split("(")[1].split(")")[0] not in TAG_COL : + return False + return True + + def __check_sma_watermark(self, arg): + if not arg: + return False + if not isinstance(arg, str): + return False + if arg[-1] not in SMA_WATMARK_MAXDELAY_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + if tdSql.get_times(arg) > WATERMARK_MAX: + return False + if tdSql.get_times(arg) < WATERMARK_MIN: + return False + + return True + + def __check_sma_max_delay(self, arg): + if not self.__check_sma_watermark(arg): + return False + if tdSql.get_times(arg) < MAX_DELAY_MIN: + return False + + return True + + def __check_sma_sliding(self, arg): + if not isinstance(arg, str): + return False + if arg[-1] not in TAOS_TIME_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + + return True + + def __get_interval_offset(self, args): + if isinstance(args, str): + interval, offset = args, None + elif isinstance(args,tuple) or isinstance(args, list): + if len(args) == 1: + interval, offset = args[0], None + elif len(args) == 2: + interval, offset = args + else: + interval, offset = False, False + else: + interval, offset = False, False + + return interval, offset + + def __check_sma_interval(self, args): + if not isinstance(args, tuple) and not isinstance(args,str): + return False + interval, offset = self.__get_interval_offset(args) + if not interval: + return False + if not self.__check_sma_sliding(interval): + return False + if tdSql.get_times(interval) < INTERVAL_MIN: + return False + if offset: + if not self.__check_sma_sliding(offset): + return False + if tdSql.get_times(interval) <= tdSql.get_times(offset) : + return False + + return True + + def __sma_create_check(self, sma:SMAschema): + if self.updatecfgDict["querySmaOptimize"] == 0: + return False + # # TODO: if database is a rollup-db, can not create sma index + # tdSql.query("select database()") + # if sma.rollup_db : + # return False tdSql.query("show stables") + if not sma.tbname: + return False stb_in_list = False for row in tdSql.queryResult: if sma.tbname == row[0]: stb_in_list = True - break if not stb_in_list: - tdSql.error(self.__create_sma_index(sma)) - if not sma.creation: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_flag: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_name: - tdSql.error(self.__create_sma_index(sma)) - if not sma.operator: - tdSql.error(self.__create_sma_index(sma)) - if not sma.tbname: - tdSql.error(self.__create_sma_index(sma)) - if not sma.func: - tdSql.error(self.__create_sma_index(sma)) - if not sma.interval: - tdSql.error(self.__create_sma_index(sma)) - if not sma.sliding: - tdSql.error(self.__create_sma_index(sma)) + return False + if not sma.creation or not isinstance(sma.creation, str) or sma.creation.upper() != "CREATE": + return False + if not sma.index_flag or not isinstance(sma.index_flag, str) or sma.index_flag.upper() != "SMA INDEX" : + return False + if not sma.index_name or not isinstance(sma.index_name, str) or sma.index_name.upper() in TAOS_KEYWORDS: + return False + if not sma.operator or not isinstance(sma.operator, str) or sma.operator.upper() != "ON": + return False + + if not sma.func or not self.__check_sma_func(sma.func): + return False + tdSql.query(f"desc {sma.tbname}") + _col_list = [] + for col_row in tdSql.queryResult: + _col_list.append(col_row[0]) + _sma_func_cols = self.__get_sma_func_col(sma.func) + for _sma_func_col in _sma_func_cols: + if _sma_func_col not in _col_list: + return False + + if sma.sliding and not self.__check_sma_sliding(sma.sliding): + return False + interval, _ = self.__get_interval_offset(sma.interval) + if not sma.interval or not self.__check_sma_interval(sma.interval) : + return False + if sma.sliding and tdSql.get_times(interval) < tdSql.get_times(sma.sliding): + return False + if sma.watermark and not self.__check_sma_watermark(sma.watermark): + return False + if sma.max_delay and not self.__check_sma_max_delay(sma.max_delay): + return False if sma.other: + return False + + return True + + def sma_create_check(self, sma:SMAschema): + if self.__sma_create_check(sma): + tdSql.query(self.__create_sma_index(sma)) + self.sma_count += 1 + self.sma_created_index.append(sma.index_name) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: tdSql.error(self.__create_sma_index(sma)) + def __drop_sma_index(self, sma:SMAschema): + sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}" + return sql + + def __sma_drop_check(self, sma:SMAschema): + if not sma.drop: + return False + if not sma.drop_flag: + return False + if not sma.index_name: + return False + + return True + + def sma_drop_check(self, sma:SMAschema): + if self.__sma_drop_check(sma): + tdSql.query(self.__drop_sma_index(sma)) + print(self.__drop_sma_index(sma)) + self.sma_count -= 1 + self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index)) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: + tdSql.error(self.__drop_sma_index(sma)) + + def __show_sma_index(self, sma:SMAschema): + sql = f"{sma.show} {sma.show_msg} {sma.show_oper} {sma.tbname}" + return sql + + def __sma_show_check(self, sma:SMAschema): + if not sma.show: + return False + if not sma.show_msg: + return False + if not sma.show_oper: + return False + if not sma.tbname: + return False + + return True + + def sma_show_check(self, sma:SMAschema): + if self.__sma_show_check(sma): + tdSql.query(self.__show_sma_index(sma)) + tdSql.checkRows(self.sma_count) + else: + tdSql.error(self.__show_sma_index(sma)) + + @property + def __create_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + # # case 1: required fields check + err_sqls.append( SMAschema(creation="", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(operator="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(tbname="", func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(func=("",),tbname=STBNAME ) ) + err_sqls.append( SMAschema(interval=(""),tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + # # case 2: err fields + err_sqls.append( SMAschema(creation="show",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="alter",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="select",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + err_sqls.append( SMAschema(index_flag="SMA INDEXES", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + # current_set + + cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + return err_sqls, cur_sqls + + def test_create_sma(self): + err_sqls , cur_sqls = self.__create_sma_sql + for err_sql in err_sqls: + self.sma_create_check(err_sql) + for cur_sql in cur_sqls: + self.sma_create_check(cur_sql) + + @property + def __drop_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + ## case 1: required fields check + err_sqls.append( SMAschema(drop="") ) + err_sqls.append( SMAschema(drop_flag="") ) + err_sqls.append( SMAschema(index_name="") ) + + for index in self.sma_created_index: + cur_sqls.append(SMAschema(index_name=index)) + + return err_sqls, cur_sqls + + def test_drop_sma(self): + err_sqls , cur_sqls = self.__drop_sma_sql + for err_sql in err_sqls: + self.sma_drop_check(err_sql) + # for cur_sql in cur_sqls: + # self.sma_drop_check(cur_sql) def all_test(self): + self.test_create_sma() + self.test_drop_sma() + pass def __create_tb(self): tdLog.printNoPrefix("==========step: create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {STBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned - ) tags (tag1 int) + ) tags ({INT_TAG} int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {NTBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, @@ -253,6 +548,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -278,7 +574,7 @@ class TDTestCase: tdSql.execute( f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") + f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") tdSql.execute( f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )") @@ -295,28 +591,31 @@ class TDTestCase: f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") def run(self): - sma1 = SMAschema(func=("min(c1)","max(c2)")) - sql1 = self.__create_sma_index(sma1) - print("================") - print(sql1) - # a = DataSet() - # return self.rows = 10 tdLog.printNoPrefix("==========step0:all check") - # self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() self.__create_tb() - self.__insert_data() + # self.__insert_data() + self.all_test() + + # drop databases, create same name db、stb and sma index + # tdSql.prepare() + # self.__create_tb() + # self.__insert_data() + # self.all_test() + + + return tdLog.printNoPrefix("==========step2:create table in rollup database") diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 244cccb041..961a6446b5 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -23,7 +23,6 @@ class TDTestCase: self.time_step = 1000 def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): - tdLog.info(" prepare datas for auto check abs function ") tdSql.execute(" create database test ") @@ -36,7 +35,7 @@ class TDTestCase: ts = self.ts for row in range(rownums): - ts += time_step*row + ts = self.ts + time_step*row c1 = random.randint(0,10000) c2 = random.randint(0,100000) c3 = random.randint(0,125) @@ -538,25 +537,41 @@ class TDTestCase: # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") tdSql.checkRows(13) - # tdSql.query("select t1 from stb1 where t1 >0 ") - # tdSql.checkRows(3) + tdSql.query("select t1 from stb1 where t1 >0 ") + tdSql.checkRows(3) + tdSql.query("select t1 from stb1 where t1 =3 ") + tdSql.checkRows(1) # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") # tdSql.checkData(0,0,61) # tdSql.query("select distinct(c1) ,t1 from stb1") # tdSql.checkRows(20) - # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") - # tdSql.checkData(0,3,33333) + tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") + tdSql.checkData(0,3,33333) # tag filter with abs function - # tdSql.query("select t1 from stb1 where abs(t1)=1") - # tdSql.checkRows(1) + tdSql.query("select t1 from stb1 where abs(t1)=1") + tdSql.checkRows(1) tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) - # tdSql.query("select t1 from stb1 where abs(t1+c1)=1") - # tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") + def support_super_table_test(self): + tdSql.execute(" use testdb ") + self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) + pass + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -593,6 +608,10 @@ class TDTestCase: self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) + tdLog.printNoPrefix("==========step8: check abs result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index e15f7675f2..edb9e25c11 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -479,6 +479,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_acos("select num1,num2 from tb3;" , "select acos(num1),acos(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_acos( " select c5 from stb1 order by ts " , "select acos(c5) from stb1 order by ts" ) + self.check_result_auto_acos( " select c5 from stb1 order by tbname " , "select acos(c5) from stb1 order by tbname" ) + self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_acos( " select t1,c5 from stb1 order by ts " , "select acos(t1), acos(c5) from stb1 order by ts" ) + self.check_result_auto_acos( " select t1,c5 from stb1 order by tbname " , "select acos(t1) ,acos(c5) from stb1 order by tbname" ) + self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) ,acos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) , acos(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -512,6 +526,14 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: acos filter query ============") + + self.abs_func_filter() + + tdLog.printNoPrefix("==========step8: check acos result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index ee134a56a8..faed5ef3c4 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -479,6 +479,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_asin("select num1,num2 from tb3;" , "select asin(num1),asin(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_asin( " select c5 from stb1 order by ts " , "select asin(c5) from stb1 order by ts" ) + self.check_result_auto_asin( " select c5 from stb1 order by tbname " , "select asin(c5) from stb1 order by tbname" ) + self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_asin( " select t1,c5 from stb1 order by ts " , "select asin(t1), asin(c5) from stb1 order by ts" ) + self.check_result_auto_asin( " select t1,c5 from stb1 order by tbname " , "select asin(t1) ,asin(c5) from stb1 order by tbname" ) + self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) ,asin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) , asin(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -512,6 +526,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check asin result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 1890c4605c..80d28b5ee5 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -476,6 +476,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3") + + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_atan( " select c5 from stb1 order by ts " , "select atan(c5) from stb1 order by ts" ) + self.check_result_auto_atan( " select c5 from stb1 order by tbname " , "select atan(c5) from stb1 order by tbname" ) + self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_atan( " select t1,c5 from stb1 order by ts " , "select atan(t1), atan(c5) from stb1 order by ts" ) + self.check_result_auto_atan( " select t1,c5 from stb1 order by tbname " , "select atan(t1) ,atan(c5) from stb1 order by tbname" ) + self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) ,atan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) , atan(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +523,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check arctan result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index 4196a5a8ce..b269b54a17 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -427,6 +427,18 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select ceil(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select ceil(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select ceil(t1), ceil(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) , ceil(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -455,6 +467,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check ceil result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index 0cb9f66937..1165d8d681 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -476,6 +476,19 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_cos( " select c5 from stb1 order by ts " , "select cos(c5) from stb1 order by ts" ) + self.check_result_auto_cos( " select c5 from stb1 order by tbname " , "select cos(c5) from stb1 order by tbname" ) + self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_cos( " select t1,c5 from stb1 order by ts " , "select cos(t1), cos(c5) from stb1 order by ts" ) + self.check_result_auto_cos( " select t1,c5 from stb1 order by tbname " , "select cos(t1) ,cos(c5) from stb1 order by tbname" ) + self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) ,cos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) , cos(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +522,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check cos result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index c955c24e05..7362191958 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -427,6 +427,19 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select floor(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select floor(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select floor(t1), floor(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select floor(t1) ,floor(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) ,floor(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) , floor(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -454,6 +467,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check floor result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 5ff11c84dd..df6390f59c 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 40da41eee7..5533cb840e 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 907ba329ee..f9d6e91199 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -670,7 +670,19 @@ class TDTestCase: tdSql.checkData(0,2,math.log(32767.000000000,2)) tdSql.checkData(0,3,math.log(63.500000000,2)) tdSql.checkData(0,4,63.999401166) - + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_log2( " select c5 from stb1 order by ts " , "select log(c5,2) from stb1 order by ts" ) + self.check_result_auto_log2( " select c5 from stb1 order by tbname " , "select log(c5,2) from stb1 order by tbname" ) + self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_log2( " select t1,c5 from stb1 order by ts " , "select log(t1,2), log(c5,2) from stb1 order by ts" ) + self.check_result_auto_log2( " select t1,c5 from stb1 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 order by tbname" ) + self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) , log(c5,2) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -707,7 +719,9 @@ class TDTestCase: self.abs_func_filter() - + tdLog.printNoPrefix("==========step9: check log result of stable query ============") + + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index 8b0137b411..c67162961b 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -606,7 +606,23 @@ class TDTestCase: tdSql.checkData(0,3,math.pow(63.500000000,2)) tdSql.checkData(0,5,None) + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_pow2( " select c5 from stb1 order by ts " , "select pow(c5,2) from stb1 order by ts" ) + self.check_result_auto_pow2( " select c5 from stb1 order by tbname " , "select pow(c5,2) from stb1 order by tbname" ) + self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_pow2( " select t1,c5 from stb1 order by ts " , "select pow(t1,2), pow(c5,2) from stb1 order by ts" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 order by tbname" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) , pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + pass + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -642,7 +658,9 @@ class TDTestCase: self.abs_func_filter() - + tdLog.printNoPrefix("==========step9: check pow result of stable query ============") + + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 223e56bce6..cc272abf42 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -432,6 +432,20 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select round(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select round(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select round(t1), round(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select round(t1) ,round(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) ,round(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) , round(c5) from stb1 where c1 > 0 order by tbname" ) + pass + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -459,6 +473,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check round result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 62d42a991e..2c4d90d3e7 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -476,6 +476,19 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_sin( " select c5 from stb1 order by ts " , "select sin(c5) from stb1 order by ts" ) + self.check_result_auto_sin( " select c5 from stb1 order by tbname " , "select sin(c5) from stb1 order by tbname" ) + self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname " , "select sin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname " , "select sin(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_sin( " select t1,c5 from stb1 order by ts " , "select sin(t1), sin(c5) from stb1 order by ts" ) + self.check_result_auto_sin( " select t1,c5 from stb1 order by tbname " , "select sin(t1) ,sin(c5) from stb1 order by tbname" ) + self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sin(t1) ,sin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sin(t1) , sin(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +522,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check sin result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 28e869e044..772056fd93 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -505,7 +505,19 @@ class TDTestCase: tdSql.checkData(0,2,math.sqrt(32767.000000000)) tdSql.checkData(0,3,math.sqrt(63.500000000)) - + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_sqrt( " select c5 from stb1 order by ts " , "select sqrt(c5) from stb1 order by ts" ) + self.check_result_auto_sqrt( " select c5 from stb1 order by tbname " , "select sqrt(c5) from stb1 order by tbname" ) + self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_sqrt( " select t1,c5 from stb1 order by ts " , "select sqrt(t1), sqrt(c5) from stb1 order by ts" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 order by tbname" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(t1) , sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -541,6 +553,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step9: check sqrt result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 79b5ac515b..f833a42b57 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),False) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index c8914d18c0..9610ffef24 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -476,6 +476,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_tan("select num1,num2 from tb3;" , "select tan(num1),tan(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_tan( " select c5 from stb1 order by ts " , "select tan(c5) from stb1 order by ts" ) + self.check_result_auto_tan( " select c5 from stb1 order by tbname " , "select tan(c5) from stb1 order by tbname" ) + self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname " , "select tan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname " , "select tan(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_tan( " select t1,c5 from stb1 order by ts " , "select tan(t1), tan(c5) from stb1 order by ts" ) + self.check_result_auto_tan( " select t1,c5 from stb1 order by tbname " , "select tan(t1) ,tan(c5) from stb1 order by tbname" ) + self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select tan(t1) ,tan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select tan(t1) , tan(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +523,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check tan result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py new file mode 100644 index 0000000000..2a2cb40c09 --- /dev/null +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -0,0 +1,113 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db2', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.asyncInsertData(paraDict) + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # start tmq consume processor + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + # tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("create some new ctb") + paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum'] + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data into new ctb") + pThread = tmqCom.asyncInsertData(paraDict) + + pThread.join() + tdLog.info("wait insert end") + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py new file mode 100644 index 0000000000..6cf849d1b9 --- /dev/null +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -0,0 +1,180 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + queryString = "select ts,c1,c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"]) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as database %s" %(topicNameList[1], paraDict['dbName']) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[2] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + tdLog.exit("2 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 788ae3474c..9254f57c40 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -159,7 +159,7 @@ class TMQCom: tdLog.debug("complete to create %s.%s" %(dbName, stbName)) return - def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): tsql.execute("use %s" %dbName) pre_create = "create table" sql = pre_create @@ -168,8 +168,10 @@ class TMQCom: tagValue = 'beijing' if (i % 2 == 0): tagValue = 'shanghai' + elif (i % 3 == 0): + tagValue = 'changsha' - sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue) if (i > 0) and (i%100 == 0): tsql.execute(sql) sql = pre_create @@ -235,7 +237,7 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return - def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): + def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " @@ -245,7 +247,7 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): - sql += " %s%d values "%(ctbPrefix,i) + sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j) @@ -254,7 +256,7 @@ class TMQCom: if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): tsql.execute(sql) if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(ctbPrefix,i) + sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx) else: sql = "insert into " #end sql @@ -354,7 +356,10 @@ class TMQCom: def threadFunctionForInsert(self, **paraDict): # create new connector for new tdSql instance in my thread newTdSql = tdCom.newTdSql() - self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + if 'ctbStartIdx' in paraDict.keys(): + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"]) + else: + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) return def asyncInsertData(self, paraDict): diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat deleted file mode 100644 index 59ddd3fb7d..0000000000 --- a/tests/system-test/fulltest.bat +++ /dev/null @@ -1,102 +0,0 @@ - -python3 .\test.py -f 0-others\taosShell.py -python3 .\test.py -f 0-others\taosShellError.py -python3 .\test.py -f 0-others\taosShellNetChk.py -python3 .\test.py -f 0-others\telemetry.py -python3 .\test.py -f 0-others\taosdMonitor.py -python3 .\test.py -f 0-others\udfTest.py -python3 .\test.py -f 0-others\udf_create.py -python3 .\test.py -f 0-others\udf_restart_taosd.py -python3 .\test.py -f 0-others\cachelast.py - -python3 .\test.py -f 0-others\user_control.py -python3 .\test.py -f 0-others\fsync.py - -python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py -python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py -python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py -@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py -python3 .\test.py -f 1-insert\alter_stable.py -python3 .\test.py -f 1-insert\alter_table.py -python3 .\test.py -f 2-query\between.py -python3 .\test.py -f 2-query\distinct.py -python3 .\test.py -f 2-query\varchar.py -python3 .\test.py -f 2-query\ltrim.py -python3 .\test.py -f 2-query\rtrim.py -python3 .\test.py -f 2-query\length.py -python3 .\test.py -f 2-query\char_length.py -python3 .\test.py -f 2-query\upper.py -python3 .\test.py -f 2-query\lower.py -python3 .\test.py -f 2-query\join.py -python3 .\test.py -f 2-query\join2.py -python3 .\test.py -f 2-query\cast.py -python3 .\test.py -f 2-query\union.py -python3 .\test.py -f 2-query\union1.py -python3 .\test.py -f 2-query\concat.py -python3 .\test.py -f 2-query\concat2.py -python3 .\test.py -f 2-query\concat_ws.py -python3 .\test.py -f 2-query\concat_ws2.py -python3 .\test.py -f 2-query\check_tsdb.py -python3 .\test.py -f 2-query\spread.py -python3 .\test.py -f 2-query\hyperloglog.py - -python3 .\test.py -f 2-query\timezone.py -python3 .\test.py -f 2-query\Now.py -python3 .\test.py -f 2-query\Today.py -python3 .\test.py -f 2-query\max.py -python3 .\test.py -f 2-query\min.py -python3 .\test.py -f 2-query\count.py -python3 .\test.py -f 2-query\last.py -python3 .\test.py -f 2-query\first.py -python3 .\test.py -f 2-query\To_iso8601.py -python3 .\test.py -f 2-query\To_unixtimestamp.py -python3 .\test.py -f 2-query\timetruncate.py -python3 .\test.py -f 2-query\diff.py -python3 .\test.py -f 2-query\Timediff.py - -python3 .\test.py -f 2-query\top.py -python3 .\test.py -f 2-query\bottom.py -python3 .\test.py -f 2-query\percentile.py -python3 .\test.py -f 2-query\apercentile.py -python3 .\test.py -f 2-query\abs.py -python3 .\test.py -f 2-query\ceil.py -python3 .\test.py -f 2-query\floor.py -python3 .\test.py -f 2-query\round.py -python3 .\test.py -f 2-query\log.py -python3 .\test.py -f 2-query\pow.py -python3 .\test.py -f 2-query\sqrt.py -python3 .\test.py -f 2-query\sin.py -python3 .\test.py -f 2-query\cos.py -python3 .\test.py -f 2-query\tan.py -python3 .\test.py -f 2-query\arcsin.py -python3 .\test.py -f 2-query\arccos.py -python3 .\test.py -f 2-query\arctan.py -python3 .\test.py -f 2-query\query_cols_tags_and_or.py -@REM # python3 .\test.py -f 2-query\nestedQuery.py -@REM # TD-15983 subquery output duplicate name column. -@REM # Please Xiangyang Guo modify the following script -@REM # python3 .\test.py -f 2-query\nestedQuery_str.py - -python3 .\test.py -f 2-query\avg.py -python3 .\test.py -f 2-query\elapsed.py -python3 .\test.py -f 2-query\csum.py -python3 .\test.py -f 2-query\mavg.py -python3 .\test.py -f 2-query\diff.py -python3 .\test.py -f 2-query\sample.py -python3 .\test.py -f 2-query\function_diff.py -python3 .\test.py -f 2-query\unique.py -python3 .\test.py -f 2-query\stateduration.py -python3 .\test.py -f 2-query\function_stateduration.py -python3 .\test.py -f 2-query\statecount.py - -python3 .\test.py -f 7-tmq\basic5.py -python3 .\test.py -f 7-tmq\subscribeDb.py -python3 .\test.py -f 7-tmq\subscribeDb0.py -python3 .\test.py -f 7-tmq\subscribeDb1.py -python3 .\test.py -f 7-tmq\subscribeStb.py -python3 .\test.py -f 7-tmq\subscribeStb0.py -python3 .\test.py -f 7-tmq\subscribeStb1.py -python3 .\test.py -f 7-tmq\subscribeStb2.py -python3 .\test.py -f 7-tmq\subscribeStb3.py -python3 .\test.py -f 7-tmq\subscribeStb4.py -python3 .\test.py -f 7-tmq\db.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 571bb166e1..2e2e390d12 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -141,6 +141,7 @@ python3 ./test.py -f 7-tmq/tmqError.py python3 ./test.py -f 7-tmq/schema.py python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index adc9e0ce28..d4032cbc0d 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -6,19 +6,23 @@ if "%1" == "full" ( echo Windows Taosd Full Test set /a exitNum=0 del /Q /F failed.txt - set caseFile="fulltest.bat" + set caseFile="fulltest.sh" if not "%2" == "" ( set caseFile="%2" ) for /F "usebackq tokens=*" %%i in (!caseFile!) do ( - for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( - set /a a+=1 - echo !a! Processing %%i - call :GetTimeSeconds !time! - set time1=!_timeTemp! - echo Start at !time! - call %%i ARG1 > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + call :CheckSkipCase %%i + if !skipCase! == false ( + set line=%%i + if "!line:~,7!" == "python3" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) ) exit !exitNum! @@ -83,3 +87,8 @@ for %%a in (%tt%) do ( ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=true ) +:goto eof \ No newline at end of file diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index cd6613b17a..2037f67089 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -339,8 +339,8 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.clientVersion = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2022 by TAOS Data, Inc. All rights reserved.\n\n"; - shell.info.promptHeader = "taos> "; - shell.info.promptContinue = " -> "; + shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; + shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; shell.info.promptSize = 6; snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index f236c1eb88..9cb3541017 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -411,8 +411,9 @@ char taosGetConsoleChar() { static char mbStr[5]; static unsigned long bufLen = 0; static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0; - if (bufLen == 0) { + while (bufLen == 0) { ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL); + if (bufLen > 0 && buf[0] == 0) bufLen = 0; bufIndex = 0; } if (mbStrLen == 0){ @@ -466,6 +467,8 @@ int32_t shellReadCommand(char *command) { } else if (c < '\033') { // Ctrl keys. TODO: Implement ctrl combinations switch (c) { + case 0: + break; case 1: // ctrl A shellPositionCursorHome(&cmd); break;