diff --git a/docs-en/14-reference/03-connector/csharp.mdx b/docs-en/14-reference/03-connector/csharp.mdx index 5eb322cf91..2d1b62fe89 100644 --- a/docs-en/14-reference/03-connector/csharp.mdx +++ b/docs-en/14-reference/03-connector/csharp.mdx @@ -48,7 +48,7 @@ Please refer to [version support list](/reference/connector#version-support) * Install the [.NET SDK](https://dotnet.microsoft.com/download) * [Nuget Client](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools) (optional installation) -* Install TDengine client driver, please refer to [Install client driver](/reference/connector#Install client driver) for details +* Install TDengine client driver, please refer to [Install client driver](/reference/connector/#install-client-driver) for details ### Install via dotnet CLI @@ -57,7 +57,7 @@ Please refer to [version support list](/reference/connector#version-support) You can reference the `TDengine.Connector` published in Nuget to the current project via the `dotnet` command under the path of the existing .NET project. -``` bash +``` dotnet add package TDengine.Connector ``` @@ -66,7 +66,7 @@ dotnet add package TDengine.Connector You can download TDengine's source code and directly reference the latest version of the TDengine.Connector library -```bash +``` git clone https://github.com/taosdata/TDengine.git cd TDengine/src/connector/C#/src/ cp -r TDengineDriver/ myProject @@ -79,7 +79,7 @@ dotnet add TDengineDriver/TDengineDriver.csproj ## Create a connection -``` C# +```csharp using TDengineDriver; namespace TDengineExample diff --git a/docs-en/14-reference/03-connector/go.mdx b/docs-en/14-reference/03-connector/go.mdx index c1e85ae4eb..8a05f2d841 100644 --- a/docs-en/14-reference/03-connector/go.mdx +++ b/docs-en/14-reference/03-connector/go.mdx @@ -55,25 +55,27 @@ A "REST connection" is a connection between the application and the TDengine ins ### Pre-installation -* Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above) -* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector#Install Client Driver) for specific steps +- Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above) +- If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector/#install-client-driver) for specific steps Configure the environment variables and check the command. -* ```go env`` -* ```gcc -v`` +* `go env` +* `gcc -v` ### Use go get to install -``go get -u github.com/taosdata/driver-go/v2@develop`` +``` +go get -u github.com/taosdata/driver-go/v2@develop +``` ### Manage with go mod 1. Initialize the project with the `go mod` command. - ``text + ```text go mod init taos-demo - ``` text + ``` 2. Introduce taosSql @@ -88,7 +90,7 @@ Configure the environment variables and check the command. ```text go mod tidy - ``` 4. + ``` 4. Run the program with `go run taos-demo` or compile the binary with the `go build` command. @@ -309,6 +311,7 @@ func main() { :::info This API is created successfully without checking permissions, but only when you execute a Query or Exec, and check if user/password/host/port is legal. + ::: * `func (db *DB) Exec(query string, args . .interface{}) (Result, error)` diff --git a/docs-en/14-reference/03-connector/java.mdx b/docs-en/14-reference/03-connector/java.mdx index 6c40f753be..ff15acf1a9 100644 --- a/docs-en/14-reference/03-connector/java.mdx +++ b/docs-en/14-reference/03-connector/java.mdx @@ -69,7 +69,7 @@ Before using Java Connector to connect to the database, the following conditions ### Install the connectors - + - [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) @@ -77,7 +77,7 @@ Before using Java Connector to connect to the database, the following conditions Add following dependency in the `pom.xml` file of your Maven project: -```xml-dtd +```xml com.taosdata.jdbc taos-jdbcdriver @@ -90,7 +90,7 @@ Add following dependency in the `pom.xml` file of your Maven project: You can build Java connector from source code after cloning the TDengine project: -```shell +``` git clone https://github.com/taosdata/taos-connector-jdbc.git cd taos-connector-jdbc mvn clean install -Dmaven.test.skip=true @@ -140,40 +140,43 @@ When you use a JDBC native connection to connect to a TDengine cluster, you can 1. Do not specify hostname and port in Java applications. -```java -public Connection getConn() throws Exception{ - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - return conn; -} -``` + ```java + public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; + } + ``` 2. specify the firstEp and the secondEp in the configuration file taos.cfg -```shell -# first fully qualified domain name (FQDN) for TDengine system -firstEp cluster_node1:6030 + ```shell + # first fully qualified domain name (FQDN) for TDengine system + firstEp cluster_node1:6030 -# second fully qualified domain name (FQDN) for TDengine system, for cluster only -secondEp cluster_node2:6030 + # second fully qualified domain name (FQDN) for TDengine system, for cluster only + secondEp cluster_node2:6030 -# default system charset -# charset UTF-8 + # default system charset + # charset UTF-8 -# system locale -# locale en_US.UTF-8 -``` + # system locale + # locale en_US.UTF-8 + ``` In the above example, JDBC uses the client's configuration file to establish a connection to a hostname `cluster_node1`, port 6030, and a database named `test`. When the firstEp node in the cluster fails, JDBC attempts to connect to the cluster using secondEp. In TDengine, as long as one node in firstEp and secondEp is valid, the connection to the cluster can be established normally. -> **Note**: The configuration file here refers to the configuration file on the machine where the application that calls the JDBC Connector is located, the default path is `/etc/taos/taos.cfg` on Linux, and the default path is `C://TDengine/cfg/taos.cfg` on Windows. +:::note +The configuration file here refers to the configuration file on the machine where the application that calls the JDBC Connector is located, the default path is `/etc/taos/taos.cfg` on Linux, and the default path is `C://TDengine/cfg/taos.cfg` on Windows. + +::: diff --git a/docs-en/14-reference/03-connector/rust.mdx b/docs-en/14-reference/03-connector/rust.mdx index cd54f35982..a5cbaeac80 100644 --- a/docs-en/14-reference/03-connector/rust.mdx +++ b/docs-en/14-reference/03-connector/rust.mdx @@ -45,15 +45,15 @@ Add the [libtaos][libtaos] dependency to the [Rust](https://rust-lang.org) proje -Add [libtaos][libtaos] to the ``Cargo.toml`'' file. +Add [libtaos][libtaos] to the `Cargo.toml` file. -``toml +```toml [dependencies] # use default feature libtaos = "*" ``` - Add [libtaos][libtaos] to the `Cargo.toml` file and enable the `rest` feature. diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 362f56c865..2e646f4769 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -37,19 +37,12 @@ enum { TMQ_MSG_TYPE__EP_RSP, }; -enum { - STREAM_TRIGGER__AT_ONCE = 1, - STREAM_TRIGGER__WINDOW_CLOSE, - STREAM_TRIGGER__BY_COUNT, - STREAM_TRIGGER__BY_BATCH_COUNT, - STREAM_TRIGGER__BY_EVENT_TIME, -}; - typedef enum EStreamType { STREAM_NORMAL = 1, STREAM_INVERT, STREAM_REPROCESS, STREAM_INVALID, + STREAM_GET_ALL, } EStreamType; typedef struct { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 30ae6c2adb..1b44b6d7ea 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -96,6 +96,7 @@ extern bool tsDeadLockKillQuery; // query client extern int32_t tsQueryPolicy; +extern int32_t tsQuerySmaOptimize; // client extern int32_t tsMinSlidingTime; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 48a677a686..d69849349c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1494,10 +1494,9 @@ typedef struct { int32_t code; } STaskDropRsp; -#define STREAM_TRIGGER_AT_ONCE_SMA 0 #define STREAM_TRIGGER_AT_ONCE 1 #define STREAM_TRIGGER_WINDOW_CLOSE 2 -#define STREAM_TRIGGER_WINDOW_CLOSE_SMA 3 +#define STREAM_TRIGGER_MAX_DELAY 3 typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -2337,24 +2336,26 @@ typedef struct { } SVgEpSet; typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - int32_t dstVgId; - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int32_t numOfVgroups; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; - SVgEpSet* pVgEpSet; -} STSma; // Time-range-wise SMA + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + int32_t dstVgId; + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + tb_uid_t dstTbUid; // for dstVgroup + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* dstTbName; // for dstVgroup + char* expr; // sma expression + char* tagsFilter; + SSchemaWrapper schemaRow; // for dstVgroup + SSchemaWrapper schemaTag; // for dstVgroup +} STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2438,27 +2439,6 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { return 0; } -typedef struct { - int64_t indexUid; - STimeWindow queryWindow; -} SVGetTsmaExpWndsReq; - -#define SMA_WNDS_EXPIRE_FLAG (0x1) -#define SMA_WNDS_IS_EXPIRE(flag) (((flag)&SMA_WNDS_EXPIRE_FLAG) != 0) -#define SMA_WNDS_SET_EXPIRE(flag) ((flag) |= SMA_WNDS_EXPIRE_FLAG) - -typedef struct { - int64_t indexUid; - int8_t flags; // 0x1 all window expired - int32_t numExpWnds; - TSKEY wndSKeys[]; -} SVGetTsmaExpWndsRsp; - -int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq); -int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq); -int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq); -int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder* pCoder, SVGetTsmaExpWndsRsp* pReq); - typedef struct { int idx; } SMCreateFullTextReq; @@ -2507,7 +2487,7 @@ typedef struct { int64_t offset; int64_t sliding; int64_t dstTbUid; - int32_t dstVgId; // for stream + int32_t dstVgId; SEpSet epSet; char* expr; } STableIndexInfo; @@ -2518,7 +2498,8 @@ typedef struct { int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp); -void tFreeSTableIndexInfo(void* pInfo); + +void tFreeSTableIndexInfo(void* pInfo); typedef struct { int8_t mqMsgType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6fb9c75e6b..743f10bd55 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -190,7 +190,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) - TD_DEF_MSG_TYPE(TDMT_VND_GET_TSMA_EXP_WNDS, "vnode-get-tsma-expired-windows", SVGetTsmaExpWndsReq, SVGetTsmaExpWndsRsp) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) @@ -221,7 +220,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL) TD_NEW_MSG_SEG(TDMT_SYNC_MSG) - TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timeout", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL) @@ -237,6 +236,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_CONFIG_CHANGE, "sync-config-change", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_SEND, "sync-snapshot-send", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_RSP, "sync-snapshot-rsp", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b0fac72172..df8a0bb7f0 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -21,13 +21,13 @@ extern "C" { #endif #include "os.h" -#include "taosdef.h" #include "query.h" -#include "tname.h" -#include "tcommon.h" +#include "taosdef.h" #include "tarray.h" +#include "tcommon.h" #include "thash.h" #include "tmsg.h" +#include "tname.h" #include "transport.h" typedef struct SCatalog SCatalog; @@ -47,8 +47,8 @@ typedef enum { } AUTH_TYPE; typedef struct SUserAuthInfo { - char user[TSDB_USER_LEN]; - char dbFName[TSDB_DB_FNAME_LEN]; + char user[TSDB_USER_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; AUTH_TYPE type; } SUserAuthInfo; @@ -59,17 +59,17 @@ typedef struct SDbInfo { } SDbInfo; typedef struct SCatalogReq { - SArray *pDbVgroup; // element is db full name - SArray *pDbCfg; // element is db full name - SArray *pDbInfo; // element is db full name - SArray *pTableMeta; // element is SNAME - SArray *pTableHash; // element is SNAME - SArray *pUdf; // element is udf name - SArray *pIndex; // element is index name - SArray *pUser; // element is SUserAuthInfo - SArray *pTableIndex; // element is SNAME + SArray* pDbVgroup; // element is db full name + SArray* pDbCfg; // element is db full name + SArray* pDbInfo; // element is db full name + SArray* pTableMeta; // element is SNAME + SArray* pTableHash; // element is SNAME + SArray* pUdf; // element is udf name + SArray* pIndex; // element is index name + SArray* pUser; // element is SUserAuthInfo + SArray* pTableIndex; // element is SNAME bool qNodeRequired; // valid qnode - bool forceUpdate; + bool forceUpdate; } SCatalogReq; typedef struct SMetaRes { @@ -78,16 +78,16 @@ typedef struct SMetaRes { } SMetaRes; typedef struct SMetaData { - SArray *pDbVgroup; // pRes = SArray* - SArray *pDbCfg; // pRes = SDbCfgInfo* - SArray *pDbInfo; // pRes = SDbInfo* - SArray *pTableMeta; // pRes = STableMeta* - SArray *pTableHash; // pRes = SVgroupInfo* - SArray *pTableIndex; // pRes = SArray* - SArray *pUdfList; // pRes = SFuncInfo* - SArray *pIndex; // pRes = SIndexInfo* - SArray *pUser; // pRes = bool* - SArray *pQnodeList; // pRes = SQueryNodeAddr* + SArray* pDbVgroup; // pRes = SArray* + SArray* pDbCfg; // pRes = SDbCfgInfo* + SArray* pDbInfo; // pRes = SDbInfo* + SArray* pTableMeta; // pRes = STableMeta* + SArray* pTableHash; // pRes = SVgroupInfo* + SArray* pTableIndex; // pRes = SArray* + SArray* pUdfList; // pRes = SFuncInfo* + SArray* pIndex; // pRes = SIndexInfo* + SArray* pUser; // pRes = bool* + SArray* pQnodeList; // pRes = SQueryNodeAddr* } SMetaData; typedef struct SCatalogCfg { @@ -104,18 +104,18 @@ typedef struct SSTableMetaVersion { uint64_t dbId; uint64_t suid; int16_t sversion; - int16_t tversion; + int16_t tversion; } SSTableMetaVersion; typedef struct SDbVgVersion { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; - int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; typedef struct STbSVersion { - char* tbFName; + char* tbFName; int32_t sver; int32_t tver; } STbSVersion; @@ -125,15 +125,15 @@ typedef struct SUserAuthVersion { int32_t version; } SUserAuthVersion; -typedef SDbCfgRsp SDbCfgInfo; +typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code); -int32_t catalogInit(SCatalogCfg *cfg); +int32_t catalogInit(SCatalogCfg* cfg); /** - * Get a cluster's catalog handle for all later operations. + * Get a cluster's catalog handle for all later operations. * @param clusterId * @param catalogHandle (output, NO need to free it) * @return error code @@ -141,14 +141,14 @@ int32_t catalogInit(SCatalogCfg *cfg); int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); /** - * Free a cluster's all catalog info, usually it's not necessary, until the application is closing. + * Free a cluster's all catalog info, usually it's not necessary, until the application is closing. * no current or future usage should be guaranteed by application * @param pCatalog (input, NO more usage) * @return error code */ void catalogFreeHandle(SCatalog* pCatalog); -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum); /** * Get a DB's all vgroup info. @@ -159,7 +159,8 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, SArray** pVgroupList); +int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, const char* pDBName, + SArray** pVgroupList); int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); @@ -170,7 +171,7 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName); int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); /** - * Get a table's meta data. + * Get a table's meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -178,10 +179,11 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetTableMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, + STableMeta** pTableMeta); /** - * Get a super table's meta data. + * Get a super table's meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -189,47 +191,47 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSe * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); - -int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); +int32_t catalogGetSTableMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, + STableMeta** pTableMeta); +int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); /** - * Force refresh DB's local cached vgroup info. + * Force refresh DB's local cached vgroup info. * @param pCtg (input, got with catalogGetHandle) * @param pTrans (input, rpc object) * @param pMgmtEps (input, mnode EPs) * @param dbFName (input, db full name) * @return error code */ -int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName); +int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* dbFName); -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables); +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, SArray* pTables); /** - * Force refresh a table's local cached meta data. + * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) * @param pTableName (input, table name) - * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) + * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ -int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); +int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, + int32_t isSTable); /** - * Force refresh a table's local cached meta data and get the new one. + * Force refresh a table's local cached meta data and get the new one. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) * @param pTableName (input, table name) - * @param pTableMeta(output, table meta data, NEED to free it by calller) - * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) + * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ -int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); - - +int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, + const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); /** * Get a table's actual vgroup, for stable it's all possible vgroup list. @@ -240,7 +242,8 @@ int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); +int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, + const SName* pTableName, SArray** pVgroupList); /** * Get a table's vgroup from its name's hash value. @@ -251,8 +254,8 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const * @param vgInfo (output, vgroup info) * @return error code */ -int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); - +int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, const SName* pName, + SVgroupInfo* vgInfo); /** * Get all meta data required in pReq. @@ -261,36 +264,40 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const * @param pMgmtEps (input, mnode EPs) * @param pReq (input, reqest info) * @param pRsp (output, response data) - * @return error code + * @return error code */ -int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); +int32_t catalogGetAllMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, + SMetaData* pRsp); -int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId); +int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, uint64_t reqId, + const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId); -int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); +int32_t catalogGetQnodeList(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion** stables, uint32_t* num); -int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion** dbs, uint32_t* num); -int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num); +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); -int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); +int32_t catalogGetDBCfg(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); -int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); +int32_t catalogGetIndexMeta(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* indexName, + SIndexInfo* pInfo); -int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pRes); +int32_t catalogGetTableIndex(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + SArray** pRes); -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo); +int32_t catalogGetUdfInfo(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo); -int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); +int32_t catalogChkAuth(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, + AUTH_TYPE type, bool* pass); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); -int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet); - -int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate); +int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); +int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate); /** * Destroy catalog and relase all resources diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index c8e803c811..8888f6ca8e 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -121,7 +121,7 @@ typedef enum EFunctionType { // internal function FUNCTION_TYPE_SELECT_VALUE, - FUNCTION_TYPE_BLOCK_DIST, // block distribution aggregate function + FUNCTION_TYPE_BLOCK_DIST, // block distribution aggregate function // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL, @@ -170,6 +170,7 @@ bool fmIsMultiResFunc(int32_t funcId); bool fmIsRepeatScanFunc(int32_t funcId); bool fmIsUserDefinedFunc(int32_t funcId); bool fmIsDistExecFunc(int32_t funcId); +bool fmIsForbidFillFunc(int32_t funcId); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index c267c89384..25369f2342 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -47,7 +47,7 @@ typedef struct SDatabaseOptions { int32_t maxRowsPerBlock; int32_t minRowsPerBlock; SNodeList* pKeep; - int32_t keep[3]; + int64_t keep[3]; int32_t pages; int32_t pagesize; char precisionStr[3]; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 90cbb7a2e6..1a3bfdbb04 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -62,6 +62,7 @@ typedef struct SScanLogicNode { int64_t watermark; int16_t tsColId; double filesFactor; + SArray* pSmaIndexes; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -303,7 +304,7 @@ typedef struct SDownstreamSourceNode { typedef struct SExchangePhysiNode { SPhysiNode node; - int32_t srcGroupId; // group id of datasource suplans + int32_t srcGroupId; // group id of datasource suplans bool singleChannel; SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; @@ -437,7 +438,6 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SArray* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index c2aa86e89f..33cc4a9c86 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -144,6 +144,7 @@ typedef struct SRealTableNode { SVgroupsInfo* pVgroupList; char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES double ratio; + SArray* pSmaIndexes; } SRealTableNode; typedef struct STempTableNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 6abd1ffa6d..ef28b832a0 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -63,7 +63,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); -int32_t qSetSTableIdForRSma(SNode* pStmt, int64_t uid); +int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); int32_t qResetStmtDataBlock(void* block, bool keepBuf); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 10ece0b219..3a77cc1e19 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -48,6 +48,7 @@ typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, TAOS_SYNC_PROPOSE_NOT_LEADER = 1, TAOS_SYNC_PROPOSE_OTHER_ERROR = 2, + TAOS_SYNC_ONLY_ONE_REPLICA = 3, } ESyncProposeCode; typedef enum { @@ -83,8 +84,10 @@ typedef struct SReConfigCbMeta { SyncTerm term; SyncTerm currentTerm; SSyncCfg oldCfg; + SSyncCfg newCfg; bool isDrop; uint64_t flag; + uint64_t seqNum; } SReConfigCbMeta; typedef struct SSnapshot { @@ -106,7 +109,7 @@ typedef struct SSyncFSM { void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); void (*FpRestoreFinishCb)(struct SSyncFSM* pFsm); - void (*FpReConfigCb)(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta); + void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta); int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); @@ -184,7 +187,6 @@ int64_t syncOpen(const SSyncInfo* pSyncInfo); void syncStart(int64_t rid); void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); ESyncState syncGetMyRole(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); @@ -194,8 +196,13 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); bool syncEnvIsStart(); const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); +int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); -int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); +int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); +int32_t syncReconfigRaw(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); + +int32_t syncLeaderTransfer(int64_t rid); +int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader); // to be moved to static void syncStartNormal(int64_t rid); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index bb50fc141c..68a33d48cb 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -398,6 +398,8 @@ typedef struct SyncSnapshotSend { SyncTerm term; SyncIndex lastIndex; // lastIndex of snapshot SyncTerm lastTerm; // lastTerm of snapshot + SyncIndex lastConfigIndex; + SSyncCfg lastConfig; SyncTerm privateTerm; int32_t seq; uint32_t dataLen; @@ -456,6 +458,36 @@ void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg); void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg); void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg); +// --------------------------------------------- +typedef struct SyncLeaderTransfer { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + /* + SRaftId srcId; + SRaftId destId; + */ + SRaftId newLeaderId; +} SyncLeaderTransfer; + +SyncLeaderTransfer* syncLeaderTransferBuild(int32_t vgId); +void syncLeaderTransferDestroy(SyncLeaderTransfer* pMsg); +void syncLeaderTransferSerialize(const SyncLeaderTransfer* pMsg, char* buf, uint32_t bufLen); +void syncLeaderTransferDeserialize(const char* buf, uint32_t len, SyncLeaderTransfer* pMsg); +char* syncLeaderTransferSerialize2(const SyncLeaderTransfer* pMsg, uint32_t* len); +SyncLeaderTransfer* syncLeaderTransferDeserialize2(const char* buf, uint32_t len); +void syncLeaderTransfer2RpcMsg(const SyncLeaderTransfer* pMsg, SRpcMsg* pRpcMsg); +void syncLeaderTransferFromRpcMsg(const SRpcMsg* pRpcMsg, SyncLeaderTransfer* pMsg); +SyncLeaderTransfer* syncLeaderTransferFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncLeaderTransfer2Json(const SyncLeaderTransfer* pMsg); +char* syncLeaderTransfer2Str(const SyncLeaderTransfer* pMsg); + +// for debug ---------------------- +void syncLeaderTransferPrint(const SyncLeaderTransfer* pMsg); +void syncLeaderTransferPrint2(char* s, const SyncLeaderTransfer* pMsg); +void syncLeaderTransferLog(const SyncLeaderTransfer* pMsg); +void syncLeaderTransferLog2(char* s, const SyncLeaderTransfer* pMsg); + // on message ---------------------- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce6a3f2ce7..03308e395f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -352,9 +352,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0619) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) -#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x061D) -#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x061E) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) @@ -655,6 +652,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2654) #define TSDB_CODE_PAR_INVALID_DELETE_WHERE TAOS_DEF_ERROR_CODE(0, 0x2655) #define TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG TAOS_DEF_ERROR_CODE(0, 0x2656) +#define TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2657) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) @@ -685,6 +683,19 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) #define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) +//tsma +#define TSDB_CODE_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x3100) +#define TSDB_CODE_TSMA_NO_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x3101) +#define TSDB_CODE_TSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3102) +#define TSDB_CODE_TSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3103) +#define TSDB_CODE_TSMA_NO_INDEX_IN_CACHE TAOS_DEF_ERROR_CODE(0, 0x3104) +#define TSDB_CODE_TSMA_RM_SKEY_IN_HASH TAOS_DEF_ERROR_CODE(0, 0x3105) + +//rsma +#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) +#define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) + + #ifdef __cplusplus } #endif diff --git a/include/util/tjson.h b/include/util/tjson.h index 84f7b81726..df433227ca 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -17,16 +17,17 @@ #define _TD_UTIL_JSON_H_ #include "os.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#define tjsonGetNumberValue(pJson, pName, val, code) \ - do { \ - uint64_t _tmp = 0; \ +#define tjsonGetNumberValue(pJson, pName, val, code) \ + do { \ + uint64_t _tmp = 0; \ code = tjsonGetBigIntValue(pJson, pName, &_tmp); \ - val = _tmp; \ + val = _tmp; \ } while (0) typedef void SJson; @@ -66,18 +67,20 @@ typedef int32_t (*FToJson)(const void* pObj, SJson* pJson); int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj); int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj); int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num); +int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArray* pArray); typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); +int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SArray** pArray, int32_t itemSize); char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); -SJson* tjsonParse(const char* pStr); -bool tjsonValidateJson(const char* pJson); +SJson* tjsonParse(const char* pStr); +bool tjsonValidateJson(const char* pJson); const char* tjsonGetError(); #ifdef __cplusplus diff --git a/include/util/tlrucache.h b/include/util/tlrucache.h new file mode 100644 index 0000000000..5aee50c42a --- /dev/null +++ b/include/util/tlrucache.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_LRUCACHE_H_ +#define _TD_UTIL_LRUCACHE_H_ + +#include "thash.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SLRUCache SLRUCache; + +typedef void (*_taos_lru_deleter_t)(const void *key, size_t keyLen, void *value); + +typedef struct LRUHandle LRUHandle; + +typedef enum { + TAOS_LRU_PRIORITY_HIGH, + TAOS_LRU_PRIORITY_LOW +} LRUPriority; + +typedef enum { + TAOS_LRU_STATUS_OK, + TAOS_LRU_STATUS_FAIL, + TAOS_LRU_STATUS_INCOMPLETE, + TAOS_LRU_STATUS_OK_OVERWRITTEN +} LRUStatus; + +SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoolRatio); +void taosLRUCacheCleanup(SLRUCache *cache); + +LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, + _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority); +LRUHandle *taosLRUCacheLookup(SLRUCache * cache, const void *key, size_t keyLen); +void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); + +void taosLRUCacheEraseUnrefEntries(SLRUCache *cache); + +bool taosLRUCacheRef(SLRUCache *cache, LRUHandle *handle); +bool taosLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRef); + +void* taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle); + +size_t taosLRUCacheGetUsage(SLRUCache *cache); +size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache); + +void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity); +size_t taosLRUCacheGetCapacity(SLRUCache *cache); + +void taosLRUCacheSetStrictCapacity(SLRUCache *cache, bool strict); +bool taosLRUCacheIsStrictCapacity(SLRUCache *cache); + +#ifdef __cplusplus +} +#endif + +#endif // _TD_UTIL_LRUCACHE_H_ diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 171f06c257..bb60624145 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -167,7 +167,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 0; + pObj->schemalessType = 1; tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 65a91fe426..dc83f7bc43 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -689,16 +689,15 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); - if (TSDB_CODE_SUCCESS == code) { code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pNodeList); - tscError("0x%"PRIx64" failed to create query plan, code:%s 0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } if (TSDB_CODE_SUCCESS == code) { schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest); } else { + tscError("0x%"PRIx64" failed to create query plan, code:%s 0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); pRequest->body.queryFp(pRequest->body.param, pRequest, code); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d1d7325909..bbf9eba363 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -17,6 +17,9 @@ #include "tname.h" #include "cJSON.h" #include "tglobal.h" +#include "osSemaphore.h" +#include "osThread.h" + //================================================================================================= #define SPACE ' ' @@ -67,6 +70,9 @@ for (int i = 1; i < keyLen; ++i) { \ #define BINARY_ADD_LEN 2 // "binary" 2 means " " #define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " + +#define MAX_RETRY_TIMES 5 +#define LINE_BATCH 20000 //================================================================================================= typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; @@ -153,8 +159,16 @@ typedef struct { int64_t endTime; } SSmlCostInfo; +typedef struct{ + SRequestObj* request; + tsem_t sem; + TdThreadSpinlock lock; +} Params; + typedef struct { int64_t id; + Params *params; + bool isLast; SMLProtocolType protocol; int8_t precision; @@ -292,19 +306,10 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { const char* errStr = taos_errstr(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); + taosMsleep(100); } taos_free_result(res); -// if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { - if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - taosMsleep(10); - } break; } case SCHEMA_ACTION_ADD_TAG: { @@ -316,19 +321,10 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { const char* errStr = taos_errstr(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + taosMsleep(100); } taos_free_result(res); -// if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { - if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - taosMsleep(10); - } break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { @@ -339,19 +335,10 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + taosMsleep(100); } taos_free_result(res); -// if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { - if (code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { - TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - taosMsleep(10); - } break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { @@ -362,19 +349,10 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + taosMsleep(100); } taos_free_result(res); -// if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { - if (code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { - TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - taosMsleep(10); - } break; } case SCHEMA_ACTION_CREATE_STABLE: { @@ -414,18 +392,10 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + taosMsleep(100); } taos_free_result(res); - if (code == TSDB_CODE_MND_STB_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - taosMsleep(10); - } break; } @@ -459,6 +429,21 @@ static int32_t smlProcessSchemaAction(SSmlHandle* info, SSchema* schemaField, SH return TSDB_CODE_SUCCESS; } +static int32_t smlCheckMeta(SSchema* schema, int32_t length, SArray* cols){ + SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + for(uint16_t i = 0; i < length; i++){ + taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + } + + for(int32_t i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv* kv = (SSmlKv*)taosArrayGetP(cols, i); + if(taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL){ + return -1; + } + } + return 0; +} + static int32_t smlModifyDBSchemas(SSmlHandle* info) { int32_t code = 0; SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); @@ -469,6 +454,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { while (tableMetaSml) { SSmlSTableMeta* sTableData = *tableMetaSml; STableMeta *pTableMeta = NULL; + bool needCheckMeta = false; // for multi thread size_t superTableLen = 0; void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); @@ -519,6 +505,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { if (code != TSDB_CODE_SUCCESS) { goto end; } + needCheckMeta = true; } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); goto end; @@ -530,6 +517,20 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, (char*)superTable); goto end; } + + if(needCheckMeta){ + code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" check tag failed. super table name %s", info->id, (char*)superTable); + goto end; + } + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" check cols failed. super table name %s", info->id, (char*)superTable); + goto end; + } + } + sTableData->tableMeta = pTableMeta; tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, tableMetaSml); @@ -541,56 +542,6 @@ end: return code; } -//========================================================================= - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ -//static void escapeSpecialCharacter(uint8_t field, const char **pos) { -// const char *cur = *pos; -// if (*cur != '\\') { -// return; -// } -// switch (field) { -// case 1: -// switch (*(cur + 1)) { -// case ',': -// case ' ': -// cur++; -// break; -// default: -// break; -// } -// break; -// case 2: -// switch (*(cur + 1)) { -// case ',': -// case ' ': -// case '=': -// cur++; -// break; -// default: -// break; -// } -// break; -// case 3: -// switch (*(cur + 1)) { -// case '"': -// case '\\': -// cur++; -// break; -// default: -// break; -// } -// break; -// default: -// break; -// } -// *pos = cur; -//} - static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ const char *pVal = kvVal->value; int32_t len = kvVal->length; @@ -1327,9 +1278,46 @@ static void smlDestroyTableInfo(SSmlHandle* info, SSmlTableInfo *tag){ taosMemoryFree(tag); } +static int32_t smlKvTimeArrayCompare(const void* key1, const void* key2) { + SArray *s1 = *(SArray **)key1; + SArray *s2 = *(SArray **)key2; + SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); + SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); + ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + if (kv1->i < kv2->i) { + return -1; + } else if (kv1->i > kv2->i) { + return 1; + } else { + return 0; + } +} + +static int32_t smlKvTimeHashCompare(const void* key1, const void* key2) { + SHashObj *s1 = *(SHashObj **)key1; + SHashObj *s2 = *(SHashObj **)key2; + SSmlKv *kv1 = (SSmlKv *)taosHashGet(s1, TS, TS_LEN); + SSmlKv *kv2 = (SSmlKv *)taosHashGet(s2, TS, TS_LEN); + ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + if (kv1->i < kv2->i) { + return -1; + } else if (kv1->i > kv2->i) { + return 1; + } else { + return 0; + } +} + static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ if(dataFormat){ - taosArrayPush(oneTable->cols, &cols); + void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GE); + if(p == NULL){ + taosArrayPush(oneTable->cols, &cols); + }else{ + taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); + } return TSDB_CODE_SUCCESS; } @@ -1342,8 +1330,13 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - taosArrayPush(oneTable->cols, &kvHash); + void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GE); + if(p == NULL){ + taosArrayPush(oneTable->cols, &kvHash); + }else{ + taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); + } return TSDB_CODE_SUCCESS; } @@ -1426,6 +1419,7 @@ static void smlDestroyInfo(SSmlHandle* info){ if(!info->dataFormat){ taosArrayDestroy(info->colsContainer); } + destroyRequest(info->pRequest); taosMemoryFreeClear(info); } @@ -2206,7 +2200,6 @@ end: return ret; } - static int32_t smlInsertData(SSmlHandle* info) { int32_t code = TSDB_CODE_SUCCESS; @@ -2236,6 +2229,7 @@ static int32_t smlInsertData(SSmlHandle* info) { code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len); if(code != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlBindData failed", info->id); return code; } oneTable = (SSmlTableInfo**)taosHashIterate(info->childTables, oneTable); @@ -2248,10 +2242,12 @@ static int32_t smlInsertData(SSmlHandle* info) { } info->cost.insertRpcTime = taosGetTimestampUs(); - launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + //launchQueryImpl(info->pRequest, info->pQuery, false, NULL); +// info->affectedRows = taos_affected_rows(info->pRequest); +// return info->pRequest->code; - info->affectedRows = taos_affected_rows(info->pRequest); - return info->pRequest->code; + launchAsyncQuery(info->pRequest, info->pQuery); + return TSDB_CODE_SUCCESS; } static void smlPrintStatisticInfo(SSmlHandle *info){ @@ -2299,7 +2295,7 @@ static int smlProcess(SSmlHandle *info, char* lines[], int numLines) { code = smlParseLine(info, lines, numLines); if (code != 0) { uError("SML:0x%"PRIx64" smlParseLine error : %s", info->id, tstrerror(code)); - goto cleanup; + return code; } info->cost.lineNum = numLines; @@ -2311,51 +2307,76 @@ static int smlProcess(SSmlHandle *info, char* lines[], int numLines) { do{ code = smlModifyDBSchemas(info); if (code == 0) break; - } while (retryNum++ < taosHashGetSize(info->superTables)); + } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); - goto cleanup; + return code; } info->cost.insertBindTime = taosGetTimestampUs(); code = smlInsertData(info); if (code != 0) { uError("SML:0x%"PRIx64" smlInsertData error : %s", info->id, tstrerror(code)); - goto cleanup; + return code; } - info->cost.endTime = taosGetTimestampUs(); -cleanup: - info->cost.code = code; - smlPrintStatisticInfo(info); return code; } -static int32_t isSchemalessDb(SSmlHandle* info){ +static int32_t isSchemalessDb(STscObj *taos){ + SCatalog* catalog = NULL; + int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); + if(code != TSDB_CODE_SUCCESS){ + uError("SML get catalog error %d", code); + return code; + } + SName name; - tNameSetDbName(&name, info->taos->acctId, info->taos->db, strlen(info->taos->db)); + tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); SDbCfgInfo pInfo = {0}; - SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SEpSet ep = getEpSet_s(&taos->pAppInfo->mgmtEp); - int32_t code = catalogGetDBCfg(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, dbFname, &pInfo); + code = catalogGetDBCfg(catalog, taos->pAppInfo->pTransporter, &ep, dbFname, &pInfo); if (code != TSDB_CODE_SUCCESS) { - info->pRequest->code = code; - smlBuildInvalidDataMsg(&info->msgBuf, "catalogGetDBCfg error, code:", tstrerror(code)); return code; } taosArrayDestroy(pInfo.pRetensions); if (!pInfo.schemaless){ - info->pRequest->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&info->msgBuf, "can not insert into schemaless db:", dbFname); return TSDB_CODE_SML_INVALID_DB_CONF; } return TSDB_CODE_SUCCESS; } +static void smlInsertCallback(void* param, void* res, int32_t code) { + SRequestObj *pRequest = (SRequestObj *)res; + SSmlHandle* info = (SSmlHandle *)param; + + uDebug("SML:0x%"PRIx64" result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); + // lock + if(code != TSDB_CODE_SUCCESS){ + taosThreadSpinLock(&info->params->lock); + info->params->request->code = code; + taosThreadSpinUnlock(&info->params->lock); + } + // unlock + + printf("SML:0x%" PRIx64 " insert finished, code: %d, total: %d\n", info->id, code, info->affectedRows); + Params *pParam = info->params; + bool isLast = info->isLast; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; + smlPrintStatisticInfo(info); + smlDestroyInfo(info); + + if(isLast){ + tsem_post(&pParam->sem); + } +} + /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -2384,48 +2405,88 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr return NULL; } - SSmlHandle* info = smlBuildSmlInfo(taos, request, (SMLProtocolType)protocol, precision); - if(!info){ - return (TAOS_RES*)request; - } + ((STscObj *)taos)->schemalessType = 1; + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + + int cnt = ceil(((double)numLines)/LINE_BATCH); + Params params; + params.request = request; + tsem_init(¶ms.sem, 0, 0); + taosThreadSpinInit(&(params.lock), 0); - info->taos->schemalessType = 1; if(request->pDb == NULL){ request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - smlBuildInvalidDataMsg(&info->msgBuf, "Database not specified", NULL); + smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if(isSchemalessDb(info) != TSDB_CODE_SUCCESS){ + if(isSchemalessDb(((STscObj *)taos)) != TSDB_CODE_SUCCESS){ request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&info->msgBuf, "Cannot write data to a non schemaless database", NULL); + smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); goto end; } if (!lines) { request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&info->msgBuf, "lines is null", NULL); + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); goto end; } if(protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL){ request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; - smlBuildInvalidDataMsg(&info->msgBuf, "protocol invalidate", NULL); + smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); goto end; } if(protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)){ request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; - smlBuildInvalidDataMsg(&info->msgBuf, "precision invalidate for line protocol", NULL); + smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL); goto end; } - info->pRequest->code = smlProcess(info, lines, numLines); + for (int i = 0; i < cnt; ++i) { + SRequestObj* req = (SRequestObj*)createRequest((STscObj *)taos, TSDB_SQL_INSERT); + if(!req){ + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error request is null"); + goto end; + } + SSmlHandle* info = smlBuildSmlInfo(taos, req, (SMLProtocolType)protocol, precision); + if(!info){ + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + goto end; + } + + int32_t perBatch = LINE_BATCH; + + if(numLines > perBatch){ + numLines -= perBatch; + info->isLast = false; + }else{ + perBatch = numLines; + numLines = 0; + info->isLast = true; + } + + info->params = ¶ms; + info->affectedRows = perBatch; + info->pRequest->body.queryFp = smlInsertCallback; + info->pRequest->body.param = info; + int32_t code = smlProcess(info, lines, perBatch); + lines += perBatch; + if (code != TSDB_CODE_SUCCESS){ + info->pRequest->body.queryFp(info, req, code); + } + } + tsem_wait(¶ms.sem); end: - info->taos->schemalessType = 0; - uDebug("result:%s", info->msgBuf.buf); - smlDestroyInfo(info); + taosThreadSpinDestroy(¶ms.lock); + tsem_destroy(¶ms.sem); +// ((STscObj *)taos)->schemalessType = 0; + ((STscObj *)taos)->schemalessType = 1; + uDebug("resultend:%s", request->msgBuf); return (TAOS_RES*)request; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 58905cac19..7d49c4206f 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -408,7 +408,7 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_ pParam->userParam = userParam; if (!async) tsem_init(&pParam->rspSem, 0, 0); - sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); + sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) goto END; sendInfo->msgInfo = (SDataBuf){ .pData = buf, @@ -704,7 +704,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { void* abuf = buf; tSerializeSCMSubscribeReq(&abuf, &req); - SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) goto FAIL; SMqSubscribeCbParam param = { @@ -1008,7 +1008,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { pParam->async = async; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { tsem_destroy(&pParam->rspSem); taosMemoryFree(pParam); @@ -1162,7 +1162,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { pParam->vgId = pVg->vgId; pParam->epoch = tmq->epoch; - SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); taosMemoryFree(pParam); diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index bb9f0ed23e..03d2b48134 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -41,7 +41,7 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) -#add_test( -# NAME smlTest -# COMMAND smlTest -#) +add_test( + NAME smlTest + COMMAND smlTest +) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index f9825c50ff..dc15c0b13d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -778,9 +778,9 @@ TEST(testCase, async_api_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - taos_query(pConn, "use test"); + taos_query(pConn, "use nest"); - TAOS_RES* pRes = taos_query(pConn, "desc abc1.tu"); + TAOS_RES* pRes = taos_query(pConn, "select NOW() from (select * from regular_table_2 where tbname in ('regular_table_2_1') and q_bigint <= 9223372036854775807 and q_tinyint <= 127 and q_bool in ( true , false) ) order by ts;"); if (taos_errno(pRes) != 0) { printf("failed, reason:%s\n", taos_errstr(pRes)); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 8137583978..4ad73a6424 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -476,22 +476,40 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(sql); } +TEST(testCase, smlGetTimestampLen_Test) { + uint8_t len = smlGetTimestampLen(0); + ASSERT_EQ(len, 1); + + len = smlGetTimestampLen(1); + ASSERT_EQ(len, 1); + + len = smlGetTimestampLen(10); + ASSERT_EQ(len, 2); + + len = smlGetTimestampLen(390); + ASSERT_EQ(len, 3); + + len = smlGetTimestampLen(-1); + ASSERT_EQ(len, 1); + + len = smlGetTimestampLen(-10); + ASSERT_EQ(len, 2); + + len = smlGetTimestampLen(-390); + ASSERT_EQ(len, 3); +} + +/* TEST(testCase, smlProcess_influx_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists inflx_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists inflx_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use inflx_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[] = { "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606401000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607402000000000", @@ -505,19 +523,20 @@ TEST(testCase, smlProcess_influx_Test) { "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=\"kk\",c4=4 1451629501000000000", "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=\"\",c4=4 1451629602000000000", }; - int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_EQ(ret, 0); + pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); // case 1 - TAOS_RES *res = taos_query(taos, "select * from t_91e0b182be80332b5c530cbf872f760e"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); + pRes = taos_query(taos, "select * from t_91e0b182be80332b5c530cbf872f760e"); + ASSERT_NE(pRes, nullptr); + int fieldNum = taos_field_count(pRes); ASSERT_EQ(fieldNum, 11); printf("fieldNum:%d\n", fieldNum); TAOS_ROW row = NULL; int32_t rowIndex = 0; - while((row = taos_fetch_row(res)) != NULL) { + while((row = taos_fetch_row(pRes)) != NULL) { int64_t ts = *(int64_t*)row[0]; double load_capacity = *(double*)row[1]; double fuel_capacity = *(double*)row[2]; @@ -546,18 +565,18 @@ TEST(testCase, smlProcess_influx_Test) { } rowIndex++; } - taos_free_result(res); + taos_free_result(pRes); // case 2 - res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); - ASSERT_NE(res, nullptr); - fieldNum = taos_field_count(res); + pRes = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); + ASSERT_NE(pRes, nullptr); + fieldNum = taos_field_count(pRes); ASSERT_EQ(fieldNum, 5); printf("fieldNum:%d\n", fieldNum); rowIndex = 0; - while((row = taos_fetch_row(res)) != NULL) { - int *length = taos_fetch_lengths(res); + while((row = taos_fetch_row(pRes)) != NULL) { + int *length = taos_fetch_lengths(pRes); int64_t ts = *(int64_t*)row[0]; double c1 = *(double*)row[1]; @@ -580,20 +599,16 @@ TEST(testCase, smlProcess_influx_Test) { } rowIndex++; } - taos_free_result(res); + taos_free_result(pRes); // case 2 - res = taos_query(taos, "show tables"); - ASSERT_NE(res, nullptr); + pRes = taos_query(taos, "show tables"); + ASSERT_NE(pRes, nullptr); - row = taos_fetch_row(res); - int rowNum = taos_affected_rows(res); + row = taos_fetch_row(pRes); + int rowNum = taos_affected_rows(pRes); ASSERT_EQ(rowNum, 5); - taos_free_result(res); - - - destroyRequest(request); - smlDestroyInfo(info); + taos_free_result(pRes); } // different types @@ -601,122 +616,79 @@ TEST(testCase, smlParseLine_error_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[] = { "measure,t1=3 c1=8", "measure,t2=3 c1=8u8" }; - int ret = smlProcess(info, (char **)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_NE(ret, 0); - destroyRequest(request); - smlDestroyInfo(info); -} - -TEST(testCase, smlGetTimestampLen_Test) { - uint8_t len = smlGetTimestampLen(0); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(390); - ASSERT_EQ(len, 3); - - len = smlGetTimestampLen(-1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(-10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(-390); - ASSERT_EQ(len, 3); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, smlProcess_telnet_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists telnet_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists telnet_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use telnet_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[] = { "sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", "sys.if.bytes.out 1479496102 1.3E3 network=tcp", " sys.procs.running 1479496100 42 host=web01 " }; - int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_EQ(ret, 0); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); // case 1 - TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); + pRes = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); + ASSERT_NE(pRes, nullptr); + int fieldNum = taos_field_count(pRes); ASSERT_EQ(fieldNum, 2); - TAOS_ROW row = taos_fetch_row(res); + TAOS_ROW row = taos_fetch_row(pRes); int64_t ts = *(int64_t*)row[0]; double c1 = *(double*)row[1]; ASSERT_EQ(ts, 1479496100000); ASSERT_EQ(c1, 42); - int rowNum = taos_affected_rows(res); + int rowNum = taos_affected_rows(pRes); ASSERT_EQ(rowNum, 1); - taos_free_result(res); + taos_free_result(pRes); // case 2 - res = taos_query(taos, "show tables"); - ASSERT_NE(res, nullptr); + pRes = taos_query(taos, "show tables"); + ASSERT_NE(pRes, nullptr); - row = taos_fetch_row(res); - rowNum = taos_affected_rows(res); + row = taos_fetch_row(pRes); + rowNum = taos_affected_rows(pRes); ASSERT_EQ(rowNum, 3); - taos_free_result(res); - - destroyRequest(request); - smlDestroyInfo(info); + taos_free_result(pRes); } TEST(testCase, smlProcess_json1_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES *pRes = taos_query(taos, "create database if not exists json_db"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists json_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use json_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql = + const char *sql[] = { "[\n" " {\n" " \"metric\": \"sys.cpu.nice\",\n" @@ -724,6 +696,7 @@ TEST(testCase, smlProcess_json1_Test) { " \"value\": 18,\n" " \"tags\": {\n" " \"host\": \"web01\",\n" + " \"id\": \"t1\",\n" " \"dc\": \"lga\"\n" " }\n" " },\n" @@ -736,55 +709,48 @@ TEST(testCase, smlProcess_json1_Test) { " \"dc\": \"lga\"\n" " }\n" " }\n" - "]"; - int ret = smlProcess(info, (char **)(&sql), 1); - ASSERT_EQ(ret, 0); + "]"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); // case 1 - TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); + pRes = taos_query(taos, "select * from t1"); + ASSERT_NE(pRes, nullptr); + int fieldNum = taos_field_count(pRes); ASSERT_EQ(fieldNum, 2); - TAOS_ROW row = taos_fetch_row(res); + TAOS_ROW row = taos_fetch_row(pRes); int64_t ts = *(int64_t*)row[0]; double c1 = *(double*)row[1]; ASSERT_EQ(ts, 1346846400000); ASSERT_EQ(c1, 18); - int rowNum = taos_affected_rows(res); + int rowNum = taos_affected_rows(pRes); ASSERT_EQ(rowNum, 1); - taos_free_result(res); + taos_free_result(pRes); // case 2 - res = taos_query(taos, "show tables"); - ASSERT_NE(res, nullptr); + pRes = taos_query(taos, "show tables"); + ASSERT_NE(pRes, nullptr); - row = taos_fetch_row(res); - rowNum = taos_affected_rows(res); + row = taos_fetch_row(pRes); + rowNum = taos_affected_rows(pRes); ASSERT_EQ(rowNum, 2); - taos_free_result(res); - - destroyRequest(request); - smlDestroyInfo(info); + taos_free_result(pRes); } TEST(testCase, smlProcess_json2_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql = + const char *sql[] = { "{\n" " \"metric\": \"meter_current0\",\n" " \"timestamp\": {\n" @@ -806,29 +772,23 @@ TEST(testCase, smlProcess_json2_Test) { " },\n" " \"id\": \"d1001\"\n" " }\n" - "}"; - int32_t ret = smlProcess(info, (char **)(&sql), -1); - ASSERT_EQ(ret, 0); - destroyRequest(request); - smlDestroyInfo(info); + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, smlProcess_json3_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql = + const char *sql[] ={ "{\n" " \"metric\": \"meter_current1\",\n" " \"timestamp\": {\n" @@ -878,29 +838,23 @@ TEST(testCase, smlProcess_json3_Test) { " },\n" " \"id\": \"d1001\"\n" " }\n" - "}"; - int32_t ret = smlProcess(info, (char **)(&sql), -1); - ASSERT_EQ(ret, 0); - destroyRequest(request); - smlDestroyInfo(info); + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, smlProcess_json4_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql = "{\n" + const char *sql[] = {"{\n" " \"metric\": \"meter_current2\",\n" " \"timestamp\": {\n" " \"value\" : 1346846500000,\n" @@ -940,18 +894,17 @@ TEST(testCase, smlProcess_json4_Test) { " \"t9\": false,\n" " \"id\": \"d1001\"\n" " }\n" - "}"; - int32_t ret = smlProcess(info, (char**)(&sql), -1); - ASSERT_EQ(ret, 0); - destroyRequest(request); - smlDestroyInfo(info); + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, smlParseTelnetLine_error_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); @@ -1000,34 +953,27 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[2] = { "sys.procs.running 1479496104000 42 host=web01", "sys.procs.running 1479496104000 42u8 host=web01" }; - int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_NE(ret, 0); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(taos_errno(pRes), 0); + taos_free_result(pRes); - destroyRequest(request); - smlDestroyInfo(info); } TEST(testCase, smlParseTelnetLine_json_error_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); @@ -1095,19 +1041,13 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[2] = { + const char *sql[] = { "[\n" " {\n" " \"metric\": \"sys.cpu.nice\",\n" @@ -1129,30 +1069,22 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { " },\n" "]", }; - int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_NE(ret, 0); - - destroyRequest(request); - smlDestroyInfo(info); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[2] = { + const char *sql[] = { "[\n" " {\n" " \"metric\": \"sys.cpu.nice\",\n" @@ -1174,90 +1106,64 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { " },\n" "]", }; - int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_NE(ret, 0); - - destroyRequest(request); - smlDestroyInfo(info); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, sml_TD15662_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES *pRes = taos_query(taos, "create database if not exists db_15662 precision 'ns'"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_15662 precision 'ns' schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use db_15662"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[] = { "hetrey c0=f,c1=127i8 1626006833639", "hetrey,t1=r c0=f,c1=127i8 1626006833640", }; - int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0])); - ASSERT_EQ(ret, 0); - - destroyRequest(request); - smlDestroyInfo(info); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, sml_TD15735_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[1] = { "{'metric': 'pekoiw', 'timestamp': {'value': 1626006833639000000, 'type': 'ns'}, 'value': {'value': False, 'type': 'bool'}, 'tags': {'t0': {'value': True, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {'value': 9223372036854775807, 'type': 'bigint'}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'binaryTagValue', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}}}", }; - int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_NE(ret, 0); - - destroyRequest(request); - smlDestroyInfo(info); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_NE(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, sml_TD15742_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists TD15742"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists TD15742 schemaless 1"); taos_free_result(pRes); pRes = taos_query(taos, "use TD15742"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[] = { "test_ms,t0=t c0=f 1626006833641", }; - int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_EQ(ret, 0); - - destroyRequest(request); - smlDestroyInfo(info); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); } TEST(testCase, sml_params_Test) { @@ -1325,8 +1231,8 @@ TEST(testCase, sml_oom_Test) { pRes = taos_query(taos, "use oom"); taos_free_result(pRes); - TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 100, TSDB_SML_LINE_PROTOCOL, 0); - ASSERT_EQ(taos_errno(res), 0); + pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1354,4 +1260,4 @@ TEST(testCase, sml_16368_Test) { pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); -} +}*/ diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cb38a3cf70..08977abd61 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -91,7 +91,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "single_stable_model", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, +// {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "retension", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update @@ -175,7 +175,7 @@ static const SSysDbTableSchema userUsersSchema[] = { }; static const SSysDbTableSchema grantsSchema[] = { - {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f8ce86a30c..b7a9ef88b6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -294,7 +294,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); - if (numOfRows == 0) { + if (numOfRows <= 0) { return numOfRows; } @@ -1239,6 +1239,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return NULL; } + if (pSrc->pData == NULL) { + continue; + } colDataAssign(pDst, pSrc, pDataBlock->info.rows); } @@ -1631,25 +1634,31 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - char tv[8] = {0}; - if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); + if (pCol->type == pColInfoData->info.type) { + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, + k); } else { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, + k); } - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k); } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 91b740fc96..fbb4f78425 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -86,6 +86,7 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; +int32_t tsQuerySmaOptimize = 1; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -113,7 +114,7 @@ int32_t tsCompatibleModel = 1; int32_t tsCountAlwaysReturnValue = 1; // 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +int32_t tsMinSlidingTime = 10; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -331,6 +332,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; @@ -541,6 +543,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; + tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7d4903696..d16ab57ea9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3877,9 +3877,10 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; - if (tEncodeI32(pCoder, pSma->numOfVgroups) < 0) return -1; if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1; if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1; + if (tEncodeI64(pCoder, pSma->dstTbUid) < 0) return -1; + if (tEncodeCStr(pCoder, pSma->dstTbName) < 0) return -1; if (tEncodeI64(pCoder, pSma->interval) < 0) return -1; if (tEncodeI64(pCoder, pSma->offset) < 0) return -1; if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1; @@ -3889,17 +3890,10 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (pSma->tagsFilterLen > 0) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } - for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1; - if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; - int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; - if (tEncodeI8(pCoder, numOfEps) < 0) return -1; - for (int32_t n = 0; n < numOfEps; ++n) { - const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; - if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1; - if (tEncodeU16(pCoder, pEp->port) < 0) return -1; - } - } + + tEncodeSSchemaWrapper(pCoder, &pSma->schemaRow); + tEncodeSSchemaWrapper(pCoder, &pSma->schemaTag); + return 0; } @@ -3907,14 +3901,15 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1; if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1; - if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1; if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1; if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; - if (tDecodeI32(pCoder, &pSma->numOfVgroups) < 0) return -1; if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->dstTbUid) < 0) return -1; + if (tDecodeCStr(pCoder, &pSma->dstTbName) < 0) return -1; if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1; if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1; if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1; @@ -3928,27 +3923,9 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { } else { pSma->tagsFilter = NULL; } - if (pSma->numOfVgroups > 0) { - pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet)); - if (!pSma->pVgEpSet) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet)); - - for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1; - int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; - for (int32_t n = 0; n < numOfEps; ++n) { - SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; - if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; - if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; - } - } - } + // only needed in dstVgroup + tDecodeSSchemaWrapperEx(pCoder, &pSma->schemaRow); + tDecodeSSchemaWrapperEx(pCoder, &pSma->schemaTag); return 0; } @@ -3991,55 +3968,6 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } -int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) { - if (tStartEncode(pCoder) < 0) return -1; - - if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; - if (tEncodeI64(pCoder, pReq->queryWindow.skey) < 0) return -1; - if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1; - - tEndEncode(pCoder); - return 0; -} - -int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->queryWindow.skey) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->queryWindow.ekey) < 0) return -1; - - tEndDecode(pCoder); - return 0; -} - -int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) { - if (tStartEncode(pCoder) < 0) return -1; - - if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; - if (tEncodeI8(pCoder, pReq->flags) < 0) return -1; - if (tEncodeI32(pCoder, pReq->numExpWnds) < 0) return -1; - for (int32_t i = 0; i < pReq->numExpWnds; ++i) { - if (tEncodeI64(pCoder, pReq->wndSKeys[i]) < 0) return -1; - } - tEndEncode(pCoder); - return 0; -} - -int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder *pCoder, SVGetTsmaExpWndsRsp *pReq) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeI8(pCoder, &pReq->flags) < 0) return -1; - if (tDecodeI32(pCoder, &pReq->numExpWnds) < 0) return -1; - for (int32_t i = 0; i < pReq->numExpWnds; ++i) { - if (tDecodeI64(pCoder, &pReq->wndSKeys[i]) < 0) return -1; - } - - tEndDecode(pCoder); - return 0; -} - int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 10ba58af29..0b59e9b6cc 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -76,22 +76,22 @@ void deltaToUtcInitOnce() { static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); -static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec); -static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec); +static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec); +static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); -static int32_t (*parseLocaltimeFp[])(char* timestr, int64_t* time, int32_t timePrec) = {parseLocaltime, +static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec) = {parseLocaltime, parseLocaltimeDst}; -int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { +int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { - return parseTimeWithTz(timestr, time, timePrec, 'T'); + return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else if (checkTzPresent(timestr, len)) { - return parseTimeWithTz(timestr, time, timePrec, 0); + return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, time, timePrec); + return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec); } } @@ -309,12 +309,36 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch return 0; } -int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { +static FORCE_INLINE bool validateTm(struct tm* pTm) { + if (pTm == NULL) { + return false; + } + + int32_t dayOfMonth[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + + int32_t leapYearMonthDay = 29; + int32_t year = pTm->tm_year + 1900; + bool isLeapYear = ((year % 100) == 0)? ((year % 400) == 0):((year % 4) == 0); + + if (isLeapYear && (pTm->tm_mon == 1)) { + if (pTm->tm_mday > leapYearMonthDay) { + return false; + } + } else { + if (pTm->tm_mday > dayOfMonth[pTm->tm_mon]) { + return false; + } + } + + return true; +} + +int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec) { *time = 0; struct tm tm = {0}; char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); - if (str == NULL) { + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { return -1; } @@ -343,13 +367,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { return 0; } -int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { +int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec) { *time = 0; struct tm tm = {0}; tm.tm_isdst = -1; char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); - if (str == NULL) { + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { return -1; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index fbd46db183..3e55469a4a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -38,9 +38,13 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { SStatusRsp statusRsp = {0}; if (pRsp->pCont != NULL && pRsp->contLen > 0 && tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - pMgmt->pData->dnodeVer = statusRsp.dnodeVer; - dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); - dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); + dTrace("status msg received from mnode, dnodeVer:%" PRId64 " saved:%" PRId64, statusRsp.dnodeVer, + pMgmt->pData->dnodeVer); + if (pMgmt->pData->dnodeVer != statusRsp.dnodeVer) { + pMgmt->pData->dnodeVer = statusRsp.dnodeVer; + dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); + dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); + } } rpcFreeCont(pRsp->pCont); tFreeSStatusRsp(&statusRsp); @@ -89,7 +93,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; SRpcMsg rpcRsp = {0}; - dTrace("send status msg to mnode"); + dTrace("send status msg to mnode, dnodeVer:%" PRId64, req.dnodeVer); SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 6f6896ff9f..096fc753b2 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -81,6 +81,13 @@ int32_t dmReadEps(SDnodeData *pData) { } pData->dnodeId = dnodeId->valueint; + cJSON *dnodeVer = cJSON_GetObjectItem(root, "dnodeVer"); + if (!dnodeVer || dnodeVer->type != cJSON_String) { + dError("failed to read %s since dnodeVer not found", file); + goto _OVER; + } + pData->dnodeVer = atoll(dnodeVer->valuestring); + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { dError("failed to read %s since clusterId not found", file); @@ -193,6 +200,7 @@ int32_t dmWriteEps(SDnodeData *pData) { len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pData->dnodeId); + len += snprintf(content + len, maxLen - len, " \"dnodeVer\": \"%" PRId64 "\",\n", pData->dnodeVer); len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pData->clusterId); len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pData->dropped); len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); @@ -224,30 +232,15 @@ int32_t dmWriteEps(SDnodeData *pData) { } pData->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s", realfile); + dDebug("successed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer); return 0; } void dmUpdateEps(SDnodeData *pData, SArray *eps) { - int32_t numOfEps = taosArrayGetSize(eps); - if (numOfEps <= 0) return; - taosThreadRwlockWrlock(&pData->lock); - - int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pData->dnodeEps); - if (numOfEps != numOfEpsOld) { - dDebug("new dnode list get from mnode"); - dmResetEps(pData, eps); - dmWriteEps(pData); - } else { - int32_t size = numOfEps * sizeof(SDnodeEp); - if (memcmp(pData->dnodeEps->pData, eps->pData, size) != 0) { - dDebug("new dnode list get from mnode"); - dmResetEps(pData, eps); - dmWriteEps(pData); - } - } - + dDebug("new dnode list get from mnode, dnodeVer:%" PRId64, pData->dnodeVer); + dmResetEps(pData, eps); + dmWriteEps(pData); taosThreadRwlockUnlock(&pData->lock); } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 238ac967a2..4ea92a2538 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -298,31 +298,32 @@ typedef struct { } SVgObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - char stb[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createdTime; - int64_t uid; - int64_t stbUid; - int64_t dbUid; - int8_t intervalUnit; - int8_t slidingUnit; - int8_t timezone; - int32_t dstVgId; // for stream - int64_t dstTbUid; - int64_t interval; - int64_t offset; - int64_t sliding; - int32_t exprLen; // strlen + 1 - int32_t tagsFilterLen; - int32_t sqlLen; - int32_t astLen; - int32_t numOfVgroups; - char* expr; - char* tagsFilter; - char* sql; - char* ast; - SVgEpSet* pVgEpSet; + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char dstTbName[TSDB_TABLE_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int64_t dstTbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + char* expr; + char* tagsFilter; + char* sql; + char* ast; + SSchemaWrapper schemaRow; // for dstVgroup + SSchemaWrapper schemaTag; // for dstVgroup } SSmaObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index ad3dc7f79d..525dd2c2e5 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -20,6 +20,8 @@ "C" { #endif +#include "mndInt.h" + typedef enum { TSDB_GRANT_ALL, TSDB_GRANT_TIME, diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 05aea3f68c..80fe472c56 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -29,8 +29,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); -int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr, - int32_t* pLen, double filesFactor); +int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, + int64_t watermark, double filesFactor); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 5ac9d2233f..0175e29a77 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -75,6 +75,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); +void mndTransExecute(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index c50279889e..e0ff7a70d7 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -29,6 +29,7 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +void mndSortVnodeGid(SVgObj *pVgroup); SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7aabbf1e4f..4da3c906d7 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -447,6 +447,8 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerOld = mndAcquireConsumer(pMnode, consumerId); if (pConsumerOld == NULL) { + mInfo("receive subscribe request from new consumer: %ld", consumerId); + pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256); pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; @@ -463,7 +465,12 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } else { /*taosRLockLatch(&pConsumerOld->lock);*/ + int32_t status = atomic_load_32(&pConsumerOld->status); + + mInfo("receive subscribe request from old consumer: %ld, current status: %s", consumerId, + mndConsumerStatusName(status)); + if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; goto SUBSCRIBE_OVER; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e6c93a9bfd..b4237466d4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1424,10 +1424,10 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in char tmp[128] = {0}; int32_t len = 0; if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, + len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); } else { - len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, + len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2); } @@ -1503,8 +1503,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusB, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); char *p = buildRetension(pDb->cfg.pRetensions); @@ -1592,4 +1592,3 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3fab870277..73eea70195 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -385,7 +385,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); int64_t curMs = taosGetTimestampMs(); bool online = mndIsDnodeOnline(pDnode, curMs); - bool dnodeChanged = (statusReq.dnodeVer != dnodeVer); + bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool needCheck = !online || dnodeChanged || reboot; @@ -427,7 +427,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (!online) { mInfo("dnode:%d, from offline to online", pDnode->id); } else { - mDebug("dnode:%d, send dnode epset, online:%d dnode_ver:%" PRId64 ":%" PRId64 " reboot:%d", pDnode->id, online, + mDebug("dnode:%d, send dnode epset, online:%d dnodeVer:%" PRId64 ":%" PRId64 " reboot:%d", pDnode->id, online, statusReq.dnodeVer, dnodeVer, reboot); } diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 94acca5f61..d1d43c841c 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -14,19 +14,115 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" #include "mndGrant.h" -#include "mndInt.h" #include "mndShow.h" #ifndef _GRANT -static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + int32_t numOfRows = 0; + char *pWrite; + int32_t cols = 0; + char tmp[32]; + char tmp1[32]; + + if (pShow->numOfRows < 1) { + cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + const char *src = "community"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "false"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + src = "unlimited"; + STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + colDataAppend(pColInfo, numOfRows, tmp, false); + + numOfRows++; + } + + pShow->numOfRows += numOfRows; + return numOfRows; +} int32_t mndInitGrant(SMnode *pMnode) { - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant); - return TSDB_CODE_SUCCESS; + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant); + return 0; } + void mndCleanupGrant() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 27d13d66b6..d8a61461dd 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -403,6 +403,10 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + static int64_t mndTick = 0; + if (++mndTick % 10 == 1) { + mTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); + } syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 027de66b42..c5a4282b3b 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -397,17 +397,17 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { - terrno = TSDB_CODE_MND_TOO_MANY_MNODES; - goto _OVER; - } - pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; goto _OVER; } + if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { + terrno = TSDB_CODE_MND_TOO_MANY_MNODES; + goto _OVER; + } + if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { terrno = TSDB_CODE_NODE_OFFLINE; goto _OVER; @@ -597,6 +597,11 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } + if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; + } + pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 00c8bb30d0..18f2e993b2 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -183,7 +183,10 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { for (int32_t i = 0; i < commitOffsetReq.num; i++) { SMqOffset *pOffset = &commitOffsetReq.offsets[i]; + mInfo("commit offset %ld to vg %d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId, + pOffset->cgroup, pOffset->topicName); if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { + mError("submit offset to topic %s failed", pOffset->topicName); return -1; } bool create = false; @@ -192,7 +195,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOffset->topicName); if (pTopic == NULL) { terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - mError("submit offset to topic %s failed since %s", pOffset->topicName, terrstr()); + mError("submit offset to topic %s failed since %s", pOffset->topicName, terrstr()); continue; } pOffsetObj = taosMemoryMalloc(sizeof(SMqOffsetObj)); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index d07355dfa9..2454a4686d 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -42,8 +42,8 @@ static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { return 0; } -int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr, - int32_t* pLen, double filesFactor) { +int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, + int64_t watermark, double filesFactor) { SNode* pAst = NULL; SQueryPlan* pPlan = NULL; terrno = TSDB_CODE_SUCCESS; @@ -53,7 +53,7 @@ int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int goto END; } - if (qSetSTableIdForRSma(pAst, uid) < 0) { + if (qSetSTableIdForRsma(pAst, uid) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; goto END; } @@ -86,7 +86,7 @@ int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int } SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); - if (qSubPlanToString(plan, pStr, pLen) < 0) { + if (qSubPlanToString(plan, pDst, pDstLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; goto END; } @@ -127,6 +127,64 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet return 0; } +int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { + pTask->dispatchType = TASK_DISPATCH__NONE; + // sink + if (pStream->createdBy == STREAM_CREATED_BY__SMA) { + pTask->sinkType = TASK_SINK__SMA; + pTask->smaSink.smaId = pStream->smaId; + } else { + pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.stbUid = pStream->targetStbUid; + memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + } + return 0; +} + +int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { + pTask->sinkType = TASK_SINK__NONE; + if (pStream->fixedSinkVgId == 0) { + pTask->dispatchType = TASK_DISPATCH__SHUFFLE; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); + ASSERT(pDb); + + if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { + ASSERT(0); + return -1; + } + sdbRelease(pMnode->pSdb, pDb); + + SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t sz = taosArrayGetSize(pVgs); + SArray* sinkLv = taosArrayGetP(pStream->tasks, 0); + int32_t sinkLvSize = taosArrayGetSize(sinkLv); + for (int32_t i = 0; i < sz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); + for (int32_t j = 0; j < sinkLvSize; j++) { + SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); + if (pLastLevelTask->nodeId == pVgInfo->vgId) { + pVgInfo->taskId = pLastLevelTask->taskId; + ASSERT(pVgInfo->taskId != 0); + break; + } + } + } + } else { + pTask->dispatchType = TASK_DISPATCH__FIXED; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + SArray* pArray = taosArrayGetP(pStream->tasks, 0); + // one sink only + ASSERT(taosArrayGetSize(pArray) == 1); + SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); + pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; + pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; + pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; + } + return 0; +} + int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; pTask->nodeId = pVgroup->vgId; @@ -139,6 +197,7 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE || pTask->sinkType != TASK_SINK__NONE); mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_STREAM_TASK_DEPLOY, pVgroup->vgId); return 0; } @@ -182,7 +241,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { return pVgroup; } -int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; void* pIter = NULL; SArray* tasks = taosArrayGetP(pStream->tasks, 0); @@ -234,7 +293,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p return 0; } -int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); SStreamTask* pTask = tNewSStreamTask(pStream->uid); @@ -292,185 +351,66 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(totLevel <= 2); pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); - bool hasExtraSink = false; - bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; - if (totLevel == 2 || externalTargetDB) { + bool hasExtraSink = false; + bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; + SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); + ASSERT(pDbObj != NULL); + sdbRelease(pSdb, pDbObj); + + bool multiTarget = pDbObj->cfg.numOfVgroups > 1; + + if (totLevel == 2 || externalTargetDB || multiTarget) { SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); // add extra sink hasExtraSink = true; if (pStream->fixedSinkVgId == 0) { - mndAddShuffledSinkToStream(pMnode, pTrans, pStream); + mndAddShuffleSinkTasksToStream(pMnode, pTrans, pStream); } else { - mndAddFixedSinkToStream(pMnode, pTrans, pStream); + mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream); } } - for (int32_t level = 0; level < totLevel; level++) { - SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); - SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); - ASSERT(LIST_LENGTH(inner->pNodeList) == 1); + if (totLevel > 1) { + SStreamTask* pFinalTask; + // inner plan + { + SArray* taskInnerLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskInnerLevel); - SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); - - // if (level == totLevel - 1 /* or no snode */) { - if (level == totLevel - 1) { - // last level, source, must assign to vnode - // must be scan type - ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); - - // replicate task to each vnode - void* pIter = NULL; - while (1) { - SVgObj* pVgroup; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; - if (pVgroup->dbUid != pStream->dbUid) { - sdbRelease(pSdb, pVgroup); - continue; - } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); - mndAddTaskToTaskSet(taskOneLevel, pTask); - // source part - pTask->sourceType = TASK_SOURCE__SCAN; - pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; - - // sink part - if (level == 0) { - // only for inplace - pTask->sinkType = TASK_SINK__NONE; - if (!hasExtraSink) { -#if 1 - if (pStream->createdBy == STREAM_CREATED_BY__SMA) { - pTask->sinkType = TASK_SINK__SMA; - pTask->smaSink.smaId = pStream->smaId; - } else { - pTask->sinkType = TASK_SINK__TABLE; - pTask->tbSink.stbUid = pStream->targetStbUid; - memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); - pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); - } -#endif - } - } else { - pTask->sinkType = TASK_SINK__NONE; - } - - // dispatch part - if (level == 0) { - pTask->dispatchType = TASK_DISPATCH__NONE; - } else { - // add fixed ep dispatcher - int32_t lastLevel = level - 1; - ASSERT(lastLevel == 0); - if (hasExtraSink) lastLevel++; - SArray* pArray = taosArrayGetP(pStream->tasks, lastLevel); - // one merge only - ASSERT(taosArrayGetSize(pArray) == 1); - SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - /*pTask->dispatchMsgType = TDMT_VND_TASK_MERGE_EXEC;*/ - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - pTask->dispatchType = TASK_DISPATCH__FIXED; - - pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; - pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; - pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; - } - - // exec part - pTask->execType = TASK_EXEC__PIPE; - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - sdbRelease(pSdb, pVgroup); - } - } else { - // merge plan - - // TODO if has snode, assign to snode - - // else, assign to vnode + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); - SStreamTask* pTask = tNewSStreamTask(pStream->uid); - mndAddTaskToTaskSet(taskOneLevel, pTask); - // source part, currently only support multi source - pTask->sourceType = TASK_SOURCE__PIPE; - pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + pFinalTask = tNewSStreamTask(pStream->uid); + mndAddTaskToTaskSet(taskInnerLevel, pFinalTask); + // input + pFinalTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; - // sink part - pTask->sinkType = TASK_SINK__NONE; - - // dispatch part - ASSERT(hasExtraSink); - /*pTask->dispatchType = TASK_DISPATCH__NONE;*/ -#if 1 - - if (hasExtraSink) { - // add dispatcher - if (pStream->fixedSinkVgId == 0) { - pTask->dispatchType = TASK_DISPATCH__SHUFFLE; - - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); - ASSERT(pDb); - if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { - sdbRelease(pSdb, pDb); - qDestroyQueryPlan(pPlan); - return -1; - } - sdbRelease(pSdb, pDb); - - // put taskId to useDbRsp - // TODO: optimize - SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(pVgs); - SArray* sinkLv = taosArrayGetP(pStream->tasks, 0); - int32_t sinkLvSize = taosArrayGetSize(sinkLv); - for (int32_t i = 0; i < sz; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); - for (int32_t j = 0; j < sinkLvSize; j++) { - SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); - if (pLastLevelTask->nodeId == pVgInfo->vgId) { - pVgInfo->taskId = pLastLevelTask->taskId; - break; - } - } - } - } else { - pTask->dispatchType = TASK_DISPATCH__FIXED; - /*pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;*/ - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - SArray* pArray = taosArrayGetP(pStream->tasks, 0); - // one sink only - ASSERT(taosArrayGetSize(pArray) == 1); - SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; - pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; - pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; - } + // dispatch + if (mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pFinalTask) < 0) { + qDestroyQueryPlan(pPlan); + return -1; } -#endif - // exec part - pTask->execType = TASK_EXEC__MERGE; + // exec + pFinalTask->execType = TASK_EXEC__PIPE; SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->dbUid); - ASSERT(pVgroup); - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pTrans, pFinalTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } - sdbRelease(pSdb, pVgroup); } - taosArrayPush(pStream->tasks, &taskOneLevel); - } + // source plan + SArray* taskSourceLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskSourceLevel); + + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 1); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); -#if 0 - if (totLevel == 2) { void* pIter = NULL; while (1) { SVgObj* pVgroup; @@ -481,26 +421,71 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } SStreamTask* pTask = tNewSStreamTask(pStream->uid); + mndAddTaskToTaskSet(taskSourceLevel, pTask); - // source part - pTask->sourceType = TASK_SOURCE__MERGE; - pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + // input + pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; - // sink part + // add fixed vg dispatch pTask->sinkType = TASK_SINK__NONE; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + pTask->dispatchType = TASK_DISPATCH__FIXED; - // dispatch part - pTask->dispatchType = TASK_DISPATCH__NONE; + pTask->fixedEpDispatcher.taskId = pFinalTask->taskId; + pTask->fixedEpDispatcher.nodeId = pFinalTask->nodeId; + pTask->fixedEpDispatcher.epSet = pFinalTask->epSet; - // exec part - pTask->execType = TASK_EXEC__NONE; + // exec + pTask->execType = TASK_EXEC__PIPE; + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } } } -#endif - // free memory + if (totLevel == 1) { + SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskOneLevel); + + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + ASSERT(LIST_LENGTH(inner->pNodeList) == 1); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pStream->dbUid) { + sdbRelease(pSdb, pVgroup); + continue; + } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + mndAddTaskToTaskSet(taskOneLevel, pTask); + + // input + pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; + + // sink or dispatch + if (hasExtraSink) { + mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pTask); + } else { + mndAddSinkToTask(pMnode, pTrans, pStream, pTask); + } + + // exec + pTask->execType = TASK_EXEC__PIPE; + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } + } + } qDestroyQueryPlan(pPlan); - return 0; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index cde36eac58..c19b558f19 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -26,6 +26,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "parser.h" #include "tname.h" #define TSDB_SMA_VER_NUMBER 1 @@ -82,10 +83,12 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->dstTbName, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->uid, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->stbUid, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->dbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->dstTbUid, _OVER) SDB_SET_INT8(pRaw, dataPos, pSma->intervalUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pSma->slidingUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pSma->timezone, _OVER) @@ -147,10 +150,12 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->dstTbName, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->uid, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->stbUid, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->dbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->dstTbUid, _OVER) SDB_GET_INT8(pRaw, dataPos, &pSma->intervalUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &pSma->slidingUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &pSma->timezone, _OVER) @@ -260,14 +265,17 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.tagsFilterLen = pSma->tagsFilterLen; req.indexUid = pSma->uid; req.tableUid = pSma->stbUid; + req.dstVgId = pSma->dstVgId; + req.dstTbUid = pSma->dstTbUid; req.interval = pSma->interval; req.offset = pSma->offset; req.sliding = pSma->sliding; req.expr = pSma->expr; req.tagsFilter = pSma->tagsFilter; - req.numOfVgroups = pSma->numOfVgroups; - req.pVgEpSet = pSma->pVgEpSet; - + req.schemaRow = pSma->schemaRow; + req.schemaTag = pSma->schemaTag; + req.dstTbName = pSma->dstTbName; + // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); @@ -425,14 +433,30 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, mndReleaseDnode(pMnode, pDnode); // todo add sma info here - SVgEpSet *pVgEpSet = NULL; - int32_t numOfVgroups = 0; - if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) { + SNode *pAst = NULL; + if (nodesStringToNode(pSma->ast, &pAst) < 0) { return -1; } + if (qExtractResultSchema(pAst, &pSma->schemaRow.nCols, &pSma->schemaRow.pSchema) != 0) { + nodesDestroyNode(pAst); + return -1; + } + nodesDestroyNode(pAst); + pSma->schemaRow.version = 1; + + // TODO: the schemaTag generated by qExtractResultXXX later. + pSma->schemaTag.nCols = 1; + pSma->schemaTag.version = 1; + pSma->schemaTag.pSchema = taosMemoryCalloc(1, sizeof(SSchema)); + if (!pSma->schemaTag.pSchema) { + return -1; + } + pSma->schemaTag.pSchema[0].type = TSDB_DATA_TYPE_BIGINT; + pSma->schemaTag.pSchema[0].bytes = TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]; + pSma->schemaTag.pSchema[0].colId = pSma->schemaRow.nCols + PRIMARYKEY_TIMESTAMP_COL_ID; + pSma->schemaTag.pSchema[0].flags = 0; + snprintf(pSma->schemaTag.pSchema[0].name, TSDB_COL_NAME_LEN, "groupId"); - pSma->pVgEpSet = pVgEpSet; - pSma->numOfVgroups = numOfVgroups; int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); @@ -464,12 +488,15 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); smaObj.createdTime = taosGetTimestampMs(); smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; + snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "td.tsma.rst.tb.%s", pCreate->name); + memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); + smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN); smaObj.stbUid = pStb->uid; smaObj.dbUid = pStb->dbUid; smaObj.intervalUnit = pCreate->intervalUnit; smaObj.slidingUnit = pCreate->slidingUnit; smaObj.timezone = pCreate->timezone; - smaObj.dstVgId = pCreate->dstVgId; smaObj.interval = pCreate->interval; smaObj.offset = pCreate->offset; smaObj.sliding = pCreate->sliding; @@ -1087,53 +1114,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} - -static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) { - SSdb *pSdb = pMnode->pSdb; - SVgObj *pVgroup = NULL; - void *pIter = NULL; - SVgEpSet *pVgEpSet = NULL; - int32_t nAllocVgs = 16; - int32_t nVgs = 0; - - pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet)); - if (!pVgEpSet) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - while (1) { - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - if (pVgroup->dbUid != pDb->uid) { - sdbRelease(pSdb, pVgroup); - continue; - } - - if (nVgs >= nAllocVgs) { - void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet)); - if (!p) { - taosMemoryFree(pVgEpSet); - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pVgEpSet = (SVgEpSet *)p; - nAllocVgs *= 2; - } - - (pVgEpSet + nVgs)->vgId = pVgroup->vgId; - (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup); - - ++nVgs; - - sdbRelease(pSdb, pVgroup); - } - - *ppVgEpSet = pVgEpSet; - *numOfVgroups = nVgs; - - return 0; -} +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9981dc8530..ad0c913fb8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -395,13 +395,13 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, + if (mndConvertRsmaTask(&req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, pStb->pAst1, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, + if (mndConvertRsmaTask(&req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, pStb->pAst2, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7abe9e3c0d..a331534a93 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -252,8 +252,12 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast } if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) { + nodesDestroyNode(pAst); return -1; } + // free + nodesDestroyNode(pAst); + #if 0 printf("|"); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index a0daa72d9a..e0b4cc6a57 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -61,6 +61,12 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM } tsem_post(&pMgmt->syncSem); } else { + STrans *pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans != NULL) { + mndTransExecute(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + } + if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) { SSnapshotMeta sMeta = {0}; if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { @@ -96,10 +102,18 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { } } -void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { +void mndReConfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbMeta) { SMnode *pMnode = pFsm->data; SSyncMgmt *pMgmt = &pMnode->syncMgmt; +#if 0 +// send response + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + syncGetAndDelRespRpc(pMnode->syncMgmt.sync, cbMeta.seqNum, &rpcMsg.info); +#endif + pMgmt->errCode = cbMeta.code; mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term); @@ -192,14 +206,14 @@ int32_t mndInitSync(SMnode *pMnode) { return -1; } - mDebug("mnode sync is opened, id:%" PRId64, pMgmt->sync); + mDebug("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; } void mndCleanupSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncStop(pMgmt->sync); - mDebug("mnode sync is stopped, id:%" PRId64, pMgmt->sync); + mDebug("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); tsem_destroy(&pMgmt->syncSem); memset(pMgmt, 0, sizeof(SSyncMgmt)); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a689c89037..033687db3e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -52,8 +52,8 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); +static bool mndCantExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } -static void mndTransExecute(SMnode *pMnode, STrans *pTrans); static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransReq(SRpcMsg *pReq); static int32_t mndProcessKillTransReq(SRpcMsg *pReq); @@ -297,7 +297,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", pData); + // mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -330,7 +330,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", pData); + // mTrace("raw:%p, is created", action.pRaw); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -363,7 +363,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); + // mTrace("raw:%p, is created", action.pRaw); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -517,12 +517,12 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { if (pOld->stage == TRN_STAGE_COMMIT) { pOld->stage = TRN_STAGE_COMMIT_ACTION; - mTrace("trans:%d, stage from commit to commitAction", pNew->id); + mTrace("trans:%d, stage from commit to commitAction since perform update action", pNew->id); } if (pOld->stage == TRN_STAGE_ROLLBACK) { pOld->stage = TRN_STAGE_FINISHED; - mTrace("trans:%d, stage from rollback to finished", pNew->id); + mTrace("trans:%d, stage from rollback to finished since perform update action", pNew->id); } return 0; } @@ -914,7 +914,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { if (pAction->msgSent) return 0; - if (!pMnode->deploy && !mndIsMaster(pMnode)) return -1; + if (mndCantExecuteTransAction(pMnode)) return -1; int64_t signature = pTrans->id; signature = (signature << 32); @@ -1114,9 +1114,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) pTrans->lastEpset = pAction->epSet; } - if (code == 0) { - if (!pMnode->deploy && !mndIsMaster(pMnode)) break; + if (mndCantExecuteTransAction(pMnode)) break; + if (code == 0) { pTrans->code = 0; pTrans->redoActionPos++; mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), @@ -1160,6 +1160,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { code = mndTransExecuteRedoActions(pMnode, pTrans); } + if (mndCantExecuteTransAction(pMnode)) return false; + if (code == 0) { pTrans->code = 0; pTrans->stage = TRN_STAGE_COMMIT; @@ -1185,6 +1187,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { } static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { + if (mndCantExecuteTransAction(pMnode)) return false; + bool continueExec = true; int32_t code = mndTransCommit(pMnode, pTrans); @@ -1233,6 +1237,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = true; int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); + if (mndCantExecuteTransAction(pMnode)) return false; + if (code == 0) { pTrans->stage = TRN_STAGE_ROLLBACK; mDebug("trans:%d, stage from undoAction to rollback", pTrans->id); @@ -1250,6 +1256,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { } static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { + if (mndCantExecuteTransAction(pMnode)) return false; + bool continueExec = true; int32_t code = mndTransRollback(pMnode, pTrans); @@ -1284,10 +1292,11 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { +void mndTransExecute(SMnode *pMnode, STrans *pTrans) { bool continueExec = true; while (continueExec) { + mDebug("trans:%d, continue to execute, stage:%s", pTrans->id, mndTransStr(pTrans->stage)); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1c395fa767..8410ec3757 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -394,12 +394,24 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { return pArray; } +static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { return *dnode1Id >= *dnode2Id ? 1 : 0; } + static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; return d1Score >= d2Score ? 1 : 0; } +void mndSortVnodeGid(SVgObj *pVgroup) { + for (int32_t i = 0; i < pVgroup->replica; ++i) { + for (int32_t j = 0; j < pVgroup->replica - 1 - i; ++j) { + if (pVgroup->vnodeGid[j].dnodeId > pVgroup->vnodeGid[j + 1].dnodeId) { + TSWAP(pVgroup->vnodeGid[j], pVgroup->vnodeGid[j + 1]); + } + } + } +} + static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { SSdb *pSdb = pMnode->pSdb; int32_t allocedVnodes = 0; @@ -434,6 +446,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pDnode->numOfVnodes++; } + mndSortVnodeGid(pVgroup); return 0; } @@ -1035,7 +1048,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); - mDebug("trans:%d, used to drop redistribute vgId:%d", pTrans->id, pVgroup->vgId); + mDebug("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId); SVgObj newVg = {0}; memcpy(&newVg, pVgroup, sizeof(SVgObj)); @@ -1044,7 +1057,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); } - if (pNew1 != pOld1) { + if (pNew1 != NULL && pOld1 != NULL) { int32_t numOfVnodes = mndGetVnodesNum(pMnode, pNew1->id); if (numOfVnodes >= pNew1->numOfSupportVnodes) { mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes, @@ -1055,7 +1068,8 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id) != 0) goto _OVER; if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id) != 0) goto _OVER; } - if (pNew2 != pOld2) { + + if (pNew2 != NULL && pOld2 != NULL) { int32_t numOfVnodes = mndGetVnodesNum(pMnode, pNew2->id); if (numOfVnodes >= pNew2->numOfSupportVnodes) { mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes, @@ -1066,7 +1080,8 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id) != 0) goto _OVER; if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id) != 0) goto _OVER; } - if (pNew3 != pOld3) { + + if (pNew3 != NULL && pOld3 != NULL) { int32_t numOfVnodes = mndGetVnodesNum(pMnode, pNew3->id); if (numOfVnodes >= pNew3->numOfSupportVnodes) { mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes, @@ -1111,15 +1126,18 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; int32_t code = -1; int64_t curMs = taosGetTimestampMs(); + int32_t newDnodeId[3] = {0}; + int32_t oldDnodeId[3] = {0}; + int32_t newIndex = -1; + int32_t oldIndex = -1; - SRedistributeVgroupReq redReq = {0}; - if (tDeserializeSRedistributeVgroupReq(pReq->pCont, pReq->contLen, &redReq) != 0) { + SRedistributeVgroupReq req = {0}; + if (tDeserializeSRedistributeVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } - mInfo("vgId:%d, start to redistribute to dnode %d:%d:%d", redReq.vgId, redReq.dnodeId1, redReq.dnodeId2, - redReq.dnodeId3); + mInfo("vgId:%d, start to redistribute to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; @@ -1128,65 +1146,147 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { if (mndCheckNodeAuth(pUser) != 0) goto _OVER; - pVgroup = mndAcquireVgroup(pMnode, redReq.vgId); + pVgroup = mndAcquireVgroup(pMnode, req.vgId); if (pVgroup == NULL) goto _OVER; pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) goto _OVER; if (pVgroup->replica == 1) { - if (redReq.dnodeId2 != -1 || redReq.dnodeId3 != -1) { + if (req.dnodeId1 <= 0 || req.dnodeId2 > 0 || req.dnodeId3 > 0) { terrno = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; } - pNew1 = mndAcquireDnode(pMnode, redReq.dnodeId1); - pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); - if (pNew1 == NULL || pOld1 == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto _OVER; - } - if (pNew1 == pOld1) { + + if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) { terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; goto _OVER; } - if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs)) { + + pNew1 = mndAcquireDnode(pMnode, req.dnodeId1); + if (pNew1 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew1, curMs)) { terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; goto _OVER; } + + pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); + if (pOld1 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pOld1, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + code = mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, NULL, NULL, NULL, NULL); + } else if (pVgroup->replica == 3) { - if (redReq.dnodeId2 == -1 || redReq.dnodeId3 == -1) { + if (req.dnodeId1 <= 0 || req.dnodeId2 <= 0 || req.dnodeId3 <= 0) { terrno = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; } - pNew1 = mndAcquireDnode(pMnode, redReq.dnodeId1); - pNew2 = mndAcquireDnode(pMnode, redReq.dnodeId2); - pNew3 = mndAcquireDnode(pMnode, redReq.dnodeId3); - pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId); - pOld2 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[1].dnodeId); - pOld3 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[2].dnodeId); - if (pNew1 == NULL || pOld1 == NULL || pNew2 == NULL || pOld2 == NULL || pNew3 == NULL || pOld3 == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto _OVER; - } - if (pNew1 == pNew2 || pNew1 == pNew3 || pNew2 == pNew3) { + + if (req.dnodeId1 == req.dnodeId2 || req.dnodeId1 == req.dnodeId3 || req.dnodeId2 == req.dnodeId3) { terrno = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; } - bool changed = false; - if (pNew1 != pOld1 && pNew1 != pOld2 && pNew1 != pOld3) changed = true; - if (pNew2 != pOld1 && pNew2 != pOld2 && pNew2 != pOld3) changed = true; - if (pNew3 != pOld1 && pNew3 != pOld2 && pNew3 != pOld3) changed = true; - if (!changed) { + + if (req.dnodeId1 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId1 != pVgroup->vnodeGid[1].dnodeId && + req.dnodeId1 != pVgroup->vnodeGid[2].dnodeId) { + newDnodeId[++newIndex] = req.dnodeId1; + mInfo("vgId:2, dnode:%d will be added", newDnodeId[newIndex]); + } + + if (req.dnodeId2 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[1].dnodeId && + req.dnodeId2 != pVgroup->vnodeGid[2].dnodeId) { + newDnodeId[++newIndex] = req.dnodeId2; + mInfo("vgId:2, dnode:%d will be added", newDnodeId[newIndex]); + } + + if (req.dnodeId3 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[1].dnodeId && + req.dnodeId3 != pVgroup->vnodeGid[2].dnodeId) { + newDnodeId[++newIndex] = req.dnodeId3; + mInfo("vgId:2, dnode:%d will be added", newDnodeId[newIndex]); + } + + if (req.dnodeId1 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[0].dnodeId && + req.dnodeId3 != pVgroup->vnodeGid[0].dnodeId) { + oldDnodeId[++oldIndex] = pVgroup->vnodeGid[0].dnodeId; + mInfo("vgId:2, dnode:%d will be removed", oldDnodeId[oldIndex]); + } + + if (req.dnodeId1 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[1].dnodeId && + req.dnodeId3 != pVgroup->vnodeGid[1].dnodeId) { + oldDnodeId[++oldIndex] = pVgroup->vnodeGid[1].dnodeId; + mInfo("vgId:2, dnode:%d will be removed", oldDnodeId[oldIndex]); + } + + if (req.dnodeId1 != pVgroup->vnodeGid[2].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[2].dnodeId && + req.dnodeId3 != pVgroup->vnodeGid[2].dnodeId) { + oldDnodeId[++oldIndex] = pVgroup->vnodeGid[2].dnodeId; + mInfo("vgId:2, dnode:%d will be removed", oldDnodeId[oldIndex]); + } + + if (newDnodeId[0] != 0) { + pNew1 = mndAcquireDnode(pMnode, newDnodeId[0]); + if (pNew1 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew1, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + } + + if (newDnodeId[1] != 0) { + pNew2 = mndAcquireDnode(pMnode, newDnodeId[1]); + if (pNew2 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew2, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + } + + if (newDnodeId[2] != 0) { + pNew3 = mndAcquireDnode(pMnode, newDnodeId[2]); + if (pNew3 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pNew3, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + } + + if (oldDnodeId[0] != 0) { + pOld1 = mndAcquireDnode(pMnode, oldDnodeId[0]); + if (pOld1 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pOld1, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + } + + if (oldDnodeId[1] != 0) { + pOld2 = mndAcquireDnode(pMnode, oldDnodeId[1]); + if (pOld2 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pOld2, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + } + + if (oldDnodeId[2] != 0) { + pOld3 = mndAcquireDnode(pMnode, oldDnodeId[2]); + if (pOld3 == NULL) goto _OVER; + if (!mndIsDnodeOnline(pOld3, curMs)) { + terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; + goto _OVER; + } + } + + if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) { terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; goto _OVER; } - if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs) || !mndIsDnodeOnline(pNew2, curMs) || - !mndIsDnodeOnline(pOld2, curMs) || !mndIsDnodeOnline(pNew3, curMs) || !mndIsDnodeOnline(pOld3, curMs)) { - terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; - goto _OVER; - } + code = mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3); + } else { terrno = TSDB_CODE_MND_INVALID_REPLICA; goto _OVER; @@ -1196,8 +1296,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("vgId:%d, failed to redistribute to dnode %d %d %d since %s", redReq.vgId, redReq.dnodeId1, redReq.dnodeId2, - redReq.dnodeId3, terrstr()); + mError("vgId:%d, failed to redistribute to dnode %d:%d:%d since %s", req.vgId, req.dnodeId1, req.dnodeId2, + req.dnodeId3, terrstr()); } mndReleaseDnode(pMnode, pNew1); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index c11aaaaa8a..39e9c75888 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -163,7 +163,12 @@ void sdbSetApplyIndex(SSdb *pSdb, int64_t index) { pSdb->curVer = index; } void sdbSetApplyTerm(SSdb *pSdb, int64_t term) { pSdb->curTerm = term; } -void sdbSetCurConfig(SSdb *pSdb, int64_t config) { pSdb->curConfig = config; } +void sdbSetCurConfig(SSdb *pSdb, int64_t config) { + if (pSdb->curConfig != config) { + mDebug("mnode sync config set from %" PRId64 " to %" PRId64, pSdb->curConfig, config); + pSdb->curConfig = config; + } +} int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index b32abc3eaa..34f5d6f23d 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -240,7 +240,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { if (pFile == NULL) { taosMemoryFree(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to read sdb file:%s since %s", file, terrstr()); + mDebug("failed to read sdb file:%s since %s", file, terrstr()); return 0; } @@ -432,8 +432,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } else { pSdb->lastCommitVer = pSdb->curVer; pSdb->lastCommitTerm = pSdb->curTerm; - mDebug("write sdb file successfully, ver:%" PRId64 " term:%" PRId64 " file:%s", pSdb->lastCommitVer, - pSdb->lastCommitTerm, curfile); + mDebug("write sdb file successfully, index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", + pSdb->lastCommitVer, pSdb->lastCommitTerm, pSdb->curConfig, curfile); } terrno = code; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 978fd9013a..8dca589320 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -28,7 +28,6 @@ target_sources( # sma "src/sma/sma.c" - "src/sma/smaTDBImpl.c" "src/sma/smaEnv.c" "src/sma/smaOpen.c" "src/sma/smaRollup.c" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 4ca62f1de9..e9da125841 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -43,35 +43,17 @@ typedef struct SRSmaInfo SRSmaInfo; struct SSmaEnv { TdThreadRwlock lock; int8_t type; - TXN txn; - void *pPool; // SPoolMem - SDiskID did; - TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? - char *path; // relative path SSmaStat *pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) #define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_DID(env) ((env)->did) -#define SMA_ENV_ENV(env) ((env)->dbEnv) -#define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_STAT(env) ((env)->pStat) #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) struct SSmaStatItem { - /** - * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from - * Streaming Module or TSDB local persistence. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. - * - TSDB_SMA_STAT_DROPPED: 1)sma dropped - * N.B. only applicable to tsma - */ - int8_t state; // ETsdbSmaStat - SHashObj *expiredWindows; // key: skey of time window, value: version - STSma *pTSma; // cache schema + int8_t state; // ETsdbSmaStat + STSma *pTSma; // cache schema }; struct SSmaStat { @@ -84,29 +66,6 @@ struct SSmaStat { #define SMA_STAT_ITEMS(s) ((s)->smaStatItems) #define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) -struct SSmaKey { - TSKEY skey; - int64_t groupId; -}; - -typedef struct SDBFile SDBFile; - -struct SDBFile { - int32_t fid; - TTB *pDB; - char *path; -}; - -int32_t tdSmaBeginCommit(SSmaEnv *pEnv); -int32_t tdSmaEndCommit(SSmaEnv *pEnv); - -int32_t smaOpenDBEnv(TDB **ppEnv, const char *path); -int32_t smaCloseDBEnv(TDB *pEnv); -int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF); -int32_t smaCloseDBF(SDBFile *pDBF); -int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); -void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); - void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); #if 0 @@ -114,13 +73,6 @@ int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result); int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin); #endif -static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { - int32_t len = 0; - len += taosEncodeFixedI64(pData, tsKey); - len += taosEncodeFixedI64(pData, groupId); - return len; -} - int32_t tdInitSma(SSma *pSma); int32_t tdDropTSma(SSma *pSma, char *pMsg); int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid); @@ -128,13 +80,11 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); -int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType, bool onlyCheck); int32_t tdLockSma(SSma *pSma); int32_t tdUnLockSma(SSma *pSma); -int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); - static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); } static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); } @@ -219,11 +169,8 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk void *tdFreeRSmaInfo(SRSmaInfo *pInfo); int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version); -// TODO: This is the basic params, and should wrap the params to a queryHandle. -int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); - -int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e7a744748b..5a8564bfd1 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -58,7 +58,7 @@ struct STqReadHandle { SArray* pColIdList; // SArray int32_t cachedSchemaVer; - int64_t cachedSchemaUid; + int64_t cachedSchemaSuid; SSchemaWrapper* pSchemaWrapper; STSchema* pSchema; }; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9b277ede06..0c2b09a493 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -150,7 +150,6 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); -int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); @@ -227,7 +226,7 @@ struct SVnode { SQHandle* pQuery; }; -#define TD_VID(PVNODE) (PVNODE)->config.vgId +#define TD_VID(PVNODE) ((PVNODE)->config.vgId) #define VND_TSDB(vnd) ((vnd)->pTsdb) #define VND_RSMA0(vnd) ((vnd)->pTsdb) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 063a78b0c6..ebf5098a05 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -86,11 +86,15 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { int nData = 0; tb_uid_t uid = 0; + metaRLock(pMeta); + if (tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pData, &nData) == 0) { uid = *(tb_uid_t *)pData; tdbFree(pData); } + metaULock(pMeta); + return 0; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 689cd511c4..0b6a526d8c 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -34,13 +34,13 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { SMetaReader mr = {0}; // validate req + // save smaIndex metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) { -// TODO: just for pass case #if 1 - terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST; + terrno = TSDB_CODE_TSMA_ALREADY_EXIST; metaReaderClear(&mr); - return -1; + return -1; // don't goto _err; #else metaReaderClear(&mr); return 0; diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c index fd5dd080ca..98e5d7c66d 100644 --- a/source/dnode/vnode/src/sma/sma.c +++ b/source/dnode/vnode/src/sma/sma.c @@ -36,25 +36,9 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { return code; } -int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { - smaWarn("vgId:%d, update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { - smaWarn("vgId:%d, get tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdGetTSmaDaysImpl(pCfg, pCont, contLen, days)) < 0) { + if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); } smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 179f573e8d..5eec5076e8 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -151,31 +151,11 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, return NULL; } - ASSERT(path && (strlen(path) > 0)); - SMA_ENV_PATH(pEnv) = strdup(path); - if (!SMA_ENV_PATH(pEnv)) { - tdFreeSmaEnv(pEnv); - return NULL; - } - - SMA_ENV_DID(pEnv) = did; - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); return NULL; } - char aname[TSDB_FILENAME_LEN] = {0}; - tfsAbsoluteName(SMA_TFS(pSma), did, path, aname); - if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { - tdFreeSmaEnv(pEnv); - return NULL; - } - - if (!(pEnv->pPool = openPool())) { - tdFreeSmaEnv(pEnv); - return NULL; - } return pEnv; } @@ -205,10 +185,7 @@ void tdDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); taosMemoryFreeClear(pSmaEnv->pStat); - taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); - smaCloseDBEnv(pSmaEnv->dbEnv); - closePool(pSmaEnv->pPool); } } @@ -242,7 +219,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { } /** - * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew). + * 1. Lazy mode utilized when init SSmaStat to update expire window(or hungry mode when tdNew). * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if * tdInitSmaStat invoked in other multithread environment later. */ @@ -280,7 +257,6 @@ void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { if (pSmaStatItem) { tDestroyTSma(pSmaStatItem->pTSma); taosMemoryFreeClear(pSmaStatItem->pTSma); - taosHashCleanup(pSmaStatItem->expiredWindows); taosMemoryFreeClear(pSmaStatItem); } return NULL; @@ -341,7 +317,7 @@ int32_t tdUnLockSma(SSma *pSma) { return 0; } -int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType, bool onlyCheck) { SSmaEnv *pEnv = NULL; // return if already init @@ -399,63 +375,3 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { return TSDB_CODE_SUCCESS; }; - -int32_t tdSmaBeginCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - // start a new txn - tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { - smaWarn("tdSma tdb begin commit fail"); - return -1; - } - return 0; -} - -int32_t tdSmaEndCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - - // Commit current txn - if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { - smaWarn("tdSma tdb end commit fail"); - return -1; - } - tdbTxnClose(pTxn); - clearPool(pEnv->pPool); - return 0; -} - -#if 0 -/** - * @brief Get the start TS key of the last data block of one interval/sliding. - * - * @param pSma - * @param param - * @param result - * @return int32_t - * 1) Return 0 and fill the result if the check procedure is normal; - * 2) Return -1 if error occurs during the check procedure. - */ -int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) { - const char *procedure = ""; - if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { - return -1; - } - // fill the result - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Remove the tSma data files related to param between pWin. - * - * @param pSma - * @param param - * @param pWin - * @return int32_t - */ -int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) { - // for ("tSmaFiles of param-interval-sliding between pWin") { - // // remove the tSmaFile - // } - return TSDB_CODE_SUCCESS; -} -#endif diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index e738d3a408..1b34529506 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -65,7 +65,7 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -132,7 +132,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -167,13 +167,13 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui */ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { SSma *pSma = pVnode->pSma; - SMeta *pMeta = pVnode->pMeta; - SMsgCb *pMsgCb = &pVnode->msgCb; if (!pReq->rollup) { smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } + SMeta *pMeta = pVnode->pMeta; + SMsgCb *pMsgCb = &pVnode->msgCb; SRSmaParam *param = &pReq->pRSmaParam; if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { @@ -181,7 +181,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { return TSDB_CODE_SUCCESS; } - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP, false) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c deleted file mode 100644 index cac986d053..0000000000 --- a/source/dnode/vnode/src/sma/smaTDBImpl.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define ALLOW_FORBID_FUNC - -#include "sma.h" - -int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) { - int ret = 0; - - if (path == NULL) return -1; - - ret = tdbOpen(path, 4096, 256, ppEnv); // use as param - - if (ret != 0) { - smaError("failed to create tsdb db env, ret = %d", ret); - return -1; - } - - return 0; -} - -int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } - -static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { - const SSmaKey *pKey1 = (const SSmaKey *)arg1; - const SSmaKey *pKey2 = (const SSmaKey *)arg2; - - ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); - - if (pKey1->skey < pKey2->skey) { - return -1; - } else if (pKey1->skey > pKey2->skey) { - return 1; - } - if (pKey1->groupId < pKey2->groupId) { - return -1; - } else if (pKey1->groupId > pKey2->groupId) { - return 1; - } - - return 0; -} - -static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { - tdb_cmpr_fn_t compFunc; - - // Create a database - compFunc = tdSmaKeyCmpr; - if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { - return -1; - } - - return 0; -} - -static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } - -int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) { - // TEnv is shared by a group of SDBFile - if (!pEnv || !pDBF) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - // Open DBF - if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { - smaError("failed to open DBF: %s", pDBF->path); - smaCloseDBDb(pDBF->pDB); - return -1; - } - - return 0; -} - -int32_t smaCloseDBF(SDBFile *pDBF) { - int32_t ret = 0; - if (pDBF->pDB) { - ret = smaCloseDBDb(pDBF->pDB); - pDBF->pDB = NULL; - } - taosMemoryFreeClear(pDBF->path); - return ret; -} - -int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { - int32_t ret; - - printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); - ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); - if (ret < 0) { - smaError("failed to upsert tsma data into db, ret = %d", ret); - return -1; - } - - return 0; -} - -void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { - void *pVal = NULL; - int ret; - - ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); - - if (ret < 0) { - smaError("failed to get tsma data from db, ret = %d", ret); - return NULL; - } - - ASSERT(*valLen >= 0); - - // TODO: lock? - // TODO: Would the key/value be destoryed during return the data? - // TODO: How about the key is updated while value length is changed? The original value buffer would be freed - // automatically? - - return pVal; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c deleted file mode 100644 index b72be06455..0000000000 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ /dev/null @@ -1,1037 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" -#include "tsdb.h" - -typedef STsdbCfg STSmaKeepCfg; - -#undef _TEST_SMA_PRINT_DEBUG_LOG_ -#define SMA_STORAGE_TSDB_MINUTES 86400 -#define SMA_STORAGE_TSDB_TIMES 10 -#define SMA_STORAGE_SPLIT_FACTOR 144 // least records in tsma file -#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 -#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds - -#define SMA_STATE_ITEM_HASH_SLOT 32 - -typedef struct { - SSma *pSma; - SDBFile dFile; - const SArray *pDataBlocks; // sma data - int64_t interval; // interval with the precision of DB -} STSmaWriteH; - -typedef struct { - int32_t iter; - int32_t fid; -} SmaFsIter; - -typedef struct { - STsdb *pTsdb; - SSma *pSma; - SDBFile dFile; - int64_t interval; // interval with the precision of DB - int32_t blockSize; // size of SMA block item - int32_t days; - int8_t storageLevel; - SmaFsIter smaFsIter; -} STSmaReadH; - -typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma -} ESmaStorageLevel; - -// static func - -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval); -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit); -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); -// expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); - -// read data - -// implementation - -/** - * @brief - * - * @param pSmaH - * @param pSma - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { - STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma); - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval); - pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Init of tSma FS - * - * @param pReadH - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { - SSma *pSma = pSmaH->pSma; - - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - pSmaH->smaFsIter.iter = 0; - pSmaH->smaFsIter.fid = fid; - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Set and open tSma file if it has key locates in queryWin. - * - * @param pReadH - * @param param - * @param queryWin - * @return true - * @return false - */ -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { - // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; - // int32_t nSmaFs = taosArrayGetSize(smaFs); - - smaCloseDBF(&pReadH->dFile); - -#if 0 - while (pReadH->smaFsIter.iter < nSmaFs) { - void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); - if (pSmaFile) { // match(indexName, queryWindow) - // TODO: select the file by index_name ... - pReadH->dFile = pSmaFile; - ++pReadH->smaFsIter.iter; - break; - } - ++pReadH->smaFsIter.iter; - } - - if (pReadH->pDFile) { - tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); - return true; - } -#endif - - return false; -} - -/** - * @brief Approximate value for week/month/year. - * - * @param interval - * @param intervalUnit - * @param precision - * @param adjusted Interval already adjusted according to DB precision - * @return int64_t - */ -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { - if (adjusted) { - return interval; - } - - switch (intervalUnit) { - case TIME_UNIT_YEAR: // approximate value - interval *= 365 * 86400 * 1e3; - break; - case TIME_UNIT_MONTH: // approximate value - interval *= 30 * 86400 * 1e3; - break; - case TIME_UNIT_WEEK: // approximate value - interval *= 7 * 86400 * 1e3; - break; - case TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TIME_UNIT_SECOND: - interval *= 1e3; - break; - default: - break; - } - - switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second - return interval / 1e6; - } else { // ms - return interval; - } - break; - case TSDB_TIME_PRECISION_MICRO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e3; - } else { // ms - return interval * 1e3; - } - break; - case TSDB_TIME_PRECISION_NANO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval * 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval; - } else { // ms - return interval * 1e6; - } - break; - default: // ms - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e6; - } else { // ms - return interval; - } - break; - } - return interval; -} - -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->pDataBlocks = pDataBlocks; - pSmaH->dFile.fid = SMA_IVLD_FID; - return TSDB_CODE_SUCCESS; -} - -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { - if (pSmaH) { - smaCloseDBF(&pSmaH->dFile); - } -} - -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { - SSma *pSma = pSmaH->pSma; - ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); - - pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma - * @param interval Interval calculated by DB's precision - * @param storageLevel - * @return int32_t - */ -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); - int32_t daysPerFile = pCfg->days; // unit is minute - - if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); - if (minutes > SMA_STORAGE_TSDB_MINUTES) { - daysPerFile = SMA_STORAGE_TSDB_MINUTES; - } - } - - return daysPerFile; -} - -/** - * @brief Judge the tSma storage level - * - * @param pCfg - * @param interval - * @return int32_t - */ -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { - int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE); - if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) { - return SMA_STORAGE_LEVEL_DFILESET; - } - return SMA_STORAGE_LEVEL_TSDB; -} - -/** - * @brief Insert/Update Time-range-wise SMA data. - * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. - * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. - * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The - * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). - * - The destination file of one data block for some interval is determined by its start TS key. - * - * @param pSma - * @param msg - * @return int32_t - */ -int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); - const SArray *pDataBlocks = (const SArray *)msg; - int64_t testSkey = TSKEY_INITIAL_VAL; - - // TODO: destroy SSDataBlocks(msg) - - // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus - // the sma data would arrive ahead of the update-expired-window msg. - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - if (!pDataBlocks) { - terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); - return terrno; - } - - if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SSmaStatItem *pItem = NULL; - - tdRefSmaStat(pSma, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - - if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - - STSma *pTSma = pItem->pTSma; - STSmaWriteH tSmaH = {0}; - - if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval); - int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); - - char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId - char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? - void *pDataBuf = NULL; - int32_t sz = taosArrayGetSize(pDataBlocks); - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; - for (int32_t j = 0; j < rows; ++j) { - printf("|"); - TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval - void *pSmaKey = &smaKey; - bool isStartKey = false; - - int32_t tlen = 0; // reset the len - pDataBuf = &dataBuf; // reset the buf - for (int32_t k = 0; k < colNum; ++k) { - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - skey = *(TSKEY *)var; - testSkey = skey; - printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); - tdEncodeTSmaKey(groupId, skey, &pSmaKey); - } else { - printf(" %" PRIi64 " |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15d |", *(uint8_t *)var); - tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15d |", *(int8_t *)var); - tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15d |", *(int16_t *)var); - tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15d |", *(uint16_t *)var); - tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t *)var); - tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t *)var); - tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15ld |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15lu |", *(uint64_t *)var); - tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); - break; - case TSDB_DATA_TYPE_NCHAR: { - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - // TODO: add binary/varbinary - TASSERT(0); - default: - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - break; - } - } - printf("\n"); - // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { - if (tlen > 0) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index - // file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - if (fid != tSmaH.dFile.fid) { - if (tSmaH.dFile.fid != SMA_IVLD_FID) { - tdSmaEndCommit(pEnv); - smaCloseDBF(&tSmaH.dFile); - } - tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("vgId:%d, write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 - " queryKey:%" PRIi64, - SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); - if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - smaWarn("vgId:%d, open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - tdSmaBeginCommit(pEnv); - } - - if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - smaWarn("vgId:%d, insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 - " since %s", - SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); - tdSmaEndCommit(pEnv); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d, insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, - SMA_VID(pSma), indexUid, skey, groupId); - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tdResetExpiredWindow(pSma, pStat, indexUid, skey); - } else { - smaWarn("vgId:%d, invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); - } - } - } - tdSmaEndCommit(pEnv); // TODO: not commit for every insert - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; -} - -int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { - smaWarn("vgId:%d, drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -/** - * @brief Insert TSma data blocks to DB File build by B+Tree - * - * @param pSmaH - * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) - * @param keyLen - * @param pData - * @param dataLen - * @return int32_t - */ -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn) { - SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert tsma data blocks into B+Tree(TTB) - if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - smaWarn("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - uint32_t valueSize = 0; - void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); - ASSERT(data != NULL); - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d, insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); - } -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief When sma data received from stream computing, make the relative expired window valid. - * - * @param pSma - * @param pStat - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { - SSmaStatItem *pItem = NULL; - - tdRefSmaStat(pSma, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { - // pItem resides in hash buffer all the time unless drop sma index - // TODO: multithread protect - if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, - indexUid); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); - // TODO: use a standalone interface to received state upate notification from stream computing module. - /** - * @brief state - * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. - * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. - * when batch data caculation not finised) - * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. - */ - pItem->state = TSDB_SMA_STAT_OK; - } else { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d, expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); - return TSDB_CODE_FAILED; - } - - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Drop tSma data and local cache - * - insert/query reference - * @param pSma - * @param msg - * @return int32_t - */ -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - - // clear local cache - if (pEnv) { - smaDebug("vgId:%d, drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); - - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { - if (tdSmaStatIsDropped(pItem)) { - smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - - tdWLockSmaEnv(pEnv); - if (tdSmaStatIsDropped(pItem)) { - tdUnLockSmaEnv(pEnv); - smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - tdSmaStatSetDropped(pItem); - tdUnLockSmaEnv(pEnv); - - int32_t nSleep = 0; - int32_t refVal = INT32_MAX; - while (true) { - if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - smaDebug("vgId:%d, drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - break; - } - smaDebug("vgId:%d, wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - taosSsleep(1); - if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d, drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); - break; - }; - } - - tdFreeSmaStatItem(pItem); - smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); - } - } - // clear sma data files - // TODO: - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma Return the data between queryWin and fill the pData. - * @param pData - * @param indexUid - * @param pQuerySKey - * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. - * @return int32_t - */ -int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - SSmaStat *pStat = NULL; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d, getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - pStat = SMA_ENV_STAT(pEnv); - - tdRefSmaStat(pSma, pStat); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { - // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if - // it's NULL. - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_FAILED; - } - -#if 0 - int32_t nQueryWin = taosArrayGetSize(pQuerySKey); - for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - } - } -#endif - -#if 1 - int8_t smaStat = 0; - if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - smaWarn("vgId:%d, getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); - return TSDB_CODE_FAILED; - } - - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - smaDebug("vgId:%d, skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } else { - smaDebug("vgId:%d, skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } - - STSma *pTSma = pItem->pTSma; -#endif - -#if 1 - STSmaReadH tReadH = {0}; - tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); - smaCloseDBF(&tReadH.dFile); - - tdUnRefSmaStat(pSma, pStat); - - tdInitTSmaFile(&tReadH, indexUid, querySKey); - smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); - if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - smaWarn("vgId:%d, open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - int64_t queryGroupId = 0; - tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - - smaDebug("vgId:%d, get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, - *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - - void *result = NULL; - int32_t valueSize = 0; - if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - smaWarn("vgId:%d, get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); - smaCloseDBF(&tReadH.dFile); - return TSDB_CODE_FAILED; - } -#endif - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d, get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); - } -#endif - taosMemoryFreeClear(result); // TODO: fill the result to output - -#if 0 - int32_t nResult = 0; - int64_t lastKey = 0; - - while (true) { - if (nResult >= nMaxResult) { - break; - } - - // set and open the file according to the STSma param - if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { - char bTree[100] = "\0"; - while (strncmp(bTree, "has more nodes", 100) == 0) { - if (nResult >= nMaxResult) { - break; - } - // tdGetDataFromBTree(bTree, queryWin, lastKey) - // fill the pData - ++nResult; - } - } - } -#endif - // read data from file and fill the result - smaCloseDBF(&tReadH.dFile); - return TSDB_CODE_SUCCESS; -} - -int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { - SSmaCfg *pCfg = (SSmaCfg *)pMsg; - - if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { - return -1; - } - - tdTSmaAdd(pSma, 1); - return 0; -} - -int32_t tdDropTSma(SSma *pSma, char *pMsg) { -#if 0 - SVDropTSmaReq vDropSmaReq = {0}; - if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // TODO: send msg to stream computing to drop tSma - // if ((send msg to stream computing) < 0) { - // tDestroyTSma(&vCreateSmaReq); - // return -1; - // } - // - - if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - tdTSmaSub(pSma, 1); -#endif - - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -static SSmaStatItem *tdNewSmaStatItem(int8_t state) { - SSmaStatItem *pItem = NULL; - - pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); - if (!pItem) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pItem->state = state; - pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), - true, HASH_ENTRY_LOCK); - if (!pItem->expiredWindows) { - taosMemoryFreeClear(pItem); - return NULL; - } - - return pItem; -} - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (!pItem) { - // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later - pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (!pItem) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - return TSDB_CODE_FAILED; - } - - // cache smaMeta - STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); - if (!pTSma) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - smaWarn("vgId:%d, set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pTSma = pTSma; - - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - return TSDB_CODE_FAILED; - } - } else if (!(pItem = *(SSmaStatItem **)pItem)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - taosMemoryFreeClear(pItem->pTSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - smaWarn("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Update expired window according to msg from stream computing module. - * - * @param pSma - * @param msg SSubmitReq - * @return int32_t - */ -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { - // no time-range-sma, just return success - if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { - smaTrace("vgId:%d, not update expire window since no tSma", SMA_VID(pSma)); - return TSDB_CODE_SUCCESS; - } - - if (!SMA_META(pSma)) { - terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d, update expire window failed since no meta ptr", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { - smaError("vgId:%d, init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // Firstly, assume that tSma can only be created on super table/normal table. - // getActiveTimeWindow - - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - - TASSERT(pEnv && pStat && pItemsHash); - - // basic procedure - // TODO: optimization - tdRefSmaStat(pSma, pStat); - - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SInterval interval = {0}; - TSKEY lastWinSKey = INT64_MIN; - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return TSDB_CODE_FAILED; - } - - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (!pBlock) break; - - STSmaWrapper *pSW = NULL; - STSma *pTSma = NULL; - - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - while (true) { - STSRow *row = tGetSubmitBlkNext(&blkIter); - if (!row) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { - if (pSW) { - pSW = tFreeTSmaWrapper(pSW, false); - } - if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { - break; - } - if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - pTSma = pSW->tSma; - - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = SMA_TSDB_CFG(pSma)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; - } - - // TODO: process multiple tsma for one table uid - TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - - if (lastWinSKey != winSKey) { - lastWinSKey = winSKey; - if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } else { - smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); - } - } - } - - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; -} diff --git a/source/dnode/vnode/src/sma/smaTimeRange2.c b/source/dnode/vnode/src/sma/smaTimeRange2.c index 09adc1a6a2..9c613873ab 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange2.c +++ b/source/dnode/vnode/src/sma/smaTimeRange2.c @@ -30,54 +30,8 @@ typedef STsdbCfg STSmaKeepCfg; #define SMA_STATE_ITEM_HASH_SLOT 32 -typedef struct { - SSma *pSma; - SDBFile dFile; - const SArray *pDataBlocks; // sma data - int64_t interval; // interval with the precision of DB -} STSmaWriteH; - -typedef struct { - int32_t iter; - int32_t fid; -} SmaFsIter; - -typedef struct { - STsdb *pTsdb; - SSma *pSma; - SDBFile dFile; - int64_t interval; // interval with the precision of DB - int32_t blockSize; // size of SMA block item - int32_t days; - int8_t storageLevel; - SmaFsIter smaFsIter; -} STSmaReadH; - -typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma -} ESmaStorageLevel; - // static func -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval); -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit); -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); -// expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); - /** * @brief Judge the tsma file split days * @@ -87,7 +41,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); * @param days unit is minute * @return int32_t */ -int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { SDecoder coder = {0}; tDecoderInit(&coder, pCont, contLen); @@ -130,225 +84,6 @@ _err: // implementation -/** - * @brief - * - * @param pSmaH - * @param pSma - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { - STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma); - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval); - pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Init of tSma FS - * - * @param pReadH - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { - SSma *pSma = pSmaH->pSma; - - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - pSmaH->smaFsIter.iter = 0; - pSmaH->smaFsIter.fid = fid; - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Set and open tSma file if it has key locates in queryWin. - * - * @param pReadH - * @param param - * @param queryWin - * @return true - * @return false - */ -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { - // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; - // int32_t nSmaFs = taosArrayGetSize(smaFs); - - smaCloseDBF(&pReadH->dFile); - -#if 0 - while (pReadH->smaFsIter.iter < nSmaFs) { - void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); - if (pSmaFile) { // match(indexName, queryWindow) - // TODO: select the file by index_name ... - pReadH->dFile = pSmaFile; - ++pReadH->smaFsIter.iter; - break; - } - ++pReadH->smaFsIter.iter; - } - - if (pReadH->pDFile) { - tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); - return true; - } -#endif - - return false; -} - -/** - * @brief Approximate value for week/month/year. - * - * @param interval - * @param intervalUnit - * @param precision - * @param adjusted Interval already adjusted according to DB precision - * @return int64_t - */ -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { - if (adjusted) { - return interval; - } - - switch (intervalUnit) { - case TIME_UNIT_YEAR: // approximate value - interval *= 365 * 86400 * 1e3; - break; - case TIME_UNIT_MONTH: // approximate value - interval *= 30 * 86400 * 1e3; - break; - case TIME_UNIT_WEEK: // approximate value - interval *= 7 * 86400 * 1e3; - break; - case TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TIME_UNIT_SECOND: - interval *= 1e3; - break; - default: - break; - } - - switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second - return interval / 1e6; - } else { // ms - return interval; - } - break; - case TSDB_TIME_PRECISION_MICRO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e3; - } else { // ms - return interval * 1e3; - } - break; - case TSDB_TIME_PRECISION_NANO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval * 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval; - } else { // ms - return interval * 1e6; - } - break; - default: // ms - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e6; - } else { // ms - return interval; - } - break; - } - return interval; -} - -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->pDataBlocks = pDataBlocks; - pSmaH->dFile.fid = SMA_IVLD_FID; - return TSDB_CODE_SUCCESS; -} - -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { - if (pSmaH) { - smaCloseDBF(&pSmaH->dFile); - } -} - -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { - SSma *pSma = pSmaH->pSma; - ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); - - pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma - * @param interval Interval calculated by DB's precision - * @param storageLevel - * @return int32_t - */ -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); - int32_t daysPerFile = pCfg->days; // unit is minute - - if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); - if (minutes > SMA_STORAGE_TSDB_MINUTES) { - daysPerFile = SMA_STORAGE_TSDB_MINUTES; - } - } - - return daysPerFile; -} - -/** - * @brief Judge the tSma storage level - * - * @param pCfg - * @param interval - * @return int32_t - */ -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { - int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE); - if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) { - return SMA_STORAGE_LEVEL_DFILESET; - } - return SMA_STORAGE_LEVEL_TSDB; -} - /** * @brief Insert/Update Time-range-wise SMA data. * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. @@ -362,28 +97,28 @@ static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { * @return int32_t */ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + const SArray *pDataBlocks = (const SArray *)msg; - int64_t testSkey = TSKEY_INITIAL_VAL; // TODO: destroy SSDataBlocks(msg) // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus // the sma data would arrive ahead of the update-expired-window msg. - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE, false) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } if (!pDataBlocks) { terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); return terrno; } if (taosArrayGetSize(pDataBlocks) <= 0) { terrno = TSDB_CODE_INVALID_PARA; - smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); return TSDB_CODE_FAILED; } @@ -398,463 +133,15 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + terrno = TSDB_CODE_TSMA_INVALID_STAT; tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; } - STSma *pTSma = pItem->pTSma; - STSmaWriteH tSmaH = {0}; - - if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval); - int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); - - char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId - char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? - void *pDataBuf = NULL; - int32_t sz = taosArrayGetSize(pDataBlocks); - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; - for (int32_t j = 0; j < rows; ++j) { - printf("|"); - TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval - void *pSmaKey = &smaKey; - bool isStartKey = false; - - int32_t tlen = 0; // reset the len - pDataBuf = &dataBuf; // reset the buf - for (int32_t k = 0; k < colNum; ++k) { - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - skey = *(TSKEY *)var; - testSkey = skey; - printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); - tdEncodeTSmaKey(groupId, skey, &pSmaKey); - } else { - printf(" %" PRIi64 " |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15d |", *(uint8_t *)var); - tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15d |", *(int8_t *)var); - tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15d |", *(int16_t *)var); - tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15d |", *(uint16_t *)var); - tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t *)var); - tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t *)var); - tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15ld |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15lu |", *(uint64_t *)var); - tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); - break; - case TSDB_DATA_TYPE_NCHAR: { - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - // TODO: add binary/varbinary - TASSERT(0); - default: - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - break; - } - } - printf("\n"); - // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { - if (tlen > 0) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index - // file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - if (fid != tSmaH.dFile.fid) { - if (tSmaH.dFile.fid != SMA_IVLD_FID) { - tdSmaEndCommit(pEnv); - smaCloseDBF(&tSmaH.dFile); - } - tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 - " queryKey:%" PRIi64, - SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); - if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - tdSmaBeginCommit(pEnv); - } - - if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 - " since %s", - SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); - tdSmaEndCommit(pEnv); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, - SMA_VID(pSma), indexUid, skey, groupId); - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tdResetExpiredWindow(pSma, pStat, indexUid, skey); - } else { - smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); - } - } - } - tdSmaEndCommit(pEnv); // TODO: not commit for every insert - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; -} - -int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { - smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -/** - * @brief Insert TSma data blocks to DB File build by B+Tree - * - * @param pSmaH - * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) - * @param keyLen - * @param pData - * @param dataLen - * @return int32_t - */ -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn) { - SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert tsma data blocks into B+Tree(TTB) - if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - uint32_t valueSize = 0; - void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); - ASSERT(data != NULL); - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); - } -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief When sma data received from stream computing, make the relative expired window valid. - * - * @param pSma - * @param pStat - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { - SSmaStatItem *pItem = NULL; - - tdRefSmaStat(pSma, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { - // pItem resides in hash buffer all the time unless drop sma index - // TODO: multithread protect - if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, - indexUid); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); - // TODO: use a standalone interface to received state upate notification from stream computing module. - /** - * @brief state - * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. - * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. - * when batch data caculation not finised) - * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. - */ - pItem->state = TSDB_SMA_STAT_OK; - } else { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); - return TSDB_CODE_FAILED; - } - - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Drop tSma data and local cache - * - insert/query reference - * @param pSma - * @param msg - * @return int32_t - */ -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - - // clear local cache - if (pEnv) { - smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); - - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { - if (tdSmaStatIsDropped(pItem)) { - smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - - tdWLockSmaEnv(pEnv); - if (tdSmaStatIsDropped(pItem)) { - tdUnLockSmaEnv(pEnv); - smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - tdSmaStatSetDropped(pItem); - tdUnLockSmaEnv(pEnv); - - int32_t nSleep = 0; - int32_t refVal = INT32_MAX; - while (true) { - if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - break; - } - smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - taosSsleep(1); - if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); - break; - }; - } - - tdFreeSmaStatItem(pItem); - smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); - } - } - // clear sma data files - // TODO: - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma Return the data between queryWin and fill the pData. - * @param pData - * @param indexUid - * @param pQuerySKey - * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. - * @return int32_t - */ -int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - SSmaStat *pStat = NULL; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - pStat = SMA_ENV_STAT(pEnv); - - tdRefSmaStat(pSma, pStat); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { - // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if - // it's NULL. - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_FAILED; - } - -#if 0 - int32_t nQueryWin = taosArrayGetSize(pQuerySKey); - for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - } - } -#endif - -#if 1 - int8_t smaStat = 0; - if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); - return TSDB_CODE_FAILED; - } - - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } else { - smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } - STSma *pTSma = pItem->pTSma; -#endif - -#if 1 - STSmaReadH tReadH = {0}; - tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); - smaCloseDBF(&tReadH.dFile); tdUnRefSmaStat(pSma, pStat); - tdInitTSmaFile(&tReadH, indexUid, querySKey); - smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); - if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - int64_t queryGroupId = 0; - tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - - smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, - *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - - void *result = NULL; - int32_t valueSize = 0; - if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); - smaCloseDBF(&tReadH.dFile); - return TSDB_CODE_FAILED; - } -#endif - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); - } -#endif - taosMemoryFreeClear(result); // TODO: fill the result to output - -#if 0 - int32_t nResult = 0; - int64_t lastKey = 0; - - while (true) { - if (nResult >= nMaxResult) { - break; - } - - // set and open the file according to the STSma param - if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { - char bTree[100] = "\0"; - while (strncmp(bTree, "has more nodes", 100) == 0) { - if (nResult >= nMaxResult) { - break; - } - // tdGetDataFromBTree(bTree, queryWin, lastKey) - // fill the pData - ++nResult; - } - } - } -#endif - // read data from file and fill the result - smaCloseDBF(&tReadH.dFile); return TSDB_CODE_SUCCESS; } @@ -865,224 +152,19 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { return -1; } + if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { + // create stable to save tsma result in dstVgId + SVCreateStbReq pReq = {0}; + pReq.name = pCfg->dstTbName; + pReq.suid = pCfg->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + + if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { + return -1; + } + } + tdTSmaAdd(pSma, 1); return 0; -} - -int32_t tdDropTSma(SSma *pSma, char *pMsg) { -#if 0 - SVDropTSmaReq vDropSmaReq = {0}; - if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // TODO: send msg to stream computing to drop tSma - // if ((send msg to stream computing) < 0) { - // tDestroyTSma(&vCreateSmaReq); - // return -1; - // } - // - - if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - tdTSmaSub(pSma, 1); -#endif - - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -static SSmaStatItem *tdNewSmaStatItem(int8_t state) { - SSmaStatItem *pItem = NULL; - - pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); - if (!pItem) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pItem->state = state; - pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), - true, HASH_ENTRY_LOCK); - if (!pItem->expiredWindows) { - taosMemoryFreeClear(pItem); - return NULL; - } - - return pItem; -} - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (!pItem) { - // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later - pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (!pItem) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - return TSDB_CODE_FAILED; - } - - // cache smaMeta - STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); - if (!pTSma) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pTSma = pTSma; - - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - return TSDB_CODE_FAILED; - } - } else if (!(pItem = *(SSmaStatItem **)pItem)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - taosMemoryFreeClear(pItem->pTSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Update expired window according to msg from stream computing module. - * - * @param pSma - * @param msg SSubmitReq - * @return int32_t - */ -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { - // no time-range-sma, just return success - if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { - smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); - return TSDB_CODE_SUCCESS; - } - - if (!SMA_META(pSma)) { - terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { - smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // Firstly, assume that tSma can only be created on super table/normal table. - // getActiveTimeWindow - - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - - TASSERT(pEnv && pStat && pItemsHash); - - // basic procedure - // TODO: optimization - tdRefSmaStat(pSma, pStat); - - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SInterval interval = {0}; - TSKEY lastWinSKey = INT64_MIN; - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return TSDB_CODE_FAILED; - } - - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (!pBlock) break; - - STSmaWrapper *pSW = NULL; - STSma *pTSma = NULL; - - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - while (true) { - STSRow *row = tGetSubmitBlkNext(&blkIter); - if (!row) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { - if (pSW) { - pSW = tFreeTSmaWrapper(pSW, false); - } - if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { - break; - } - if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - pTSma = pSW->tSma; - - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = SMA_TSDB_CFG(pSma)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; - } - - // TODO: process multiple tsma for one table uid - TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - - if (lastWinSKey != winSKey) { - lastWinSKey = winSKey; - if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } else { - smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); - } - } - } - - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2f4d6c11c6..4a5ea49d79 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -130,7 +130,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); - ASSERT(pHandle); + /*ASSERT(pHandle);*/ + if (pHandle == NULL) { + tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, pTq->pVnode->config.vgId, + pReq->subKey); + return -1; + } + + if (pHandle->consumerId != consumerId) { + tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld", + consumerId, pTq->pVnode->config.vgId, pReq->subKey, pHandle->consumerId); + return -1; + } int32_t consumerEpoch = atomic_load_32(&pHandle->epoch); while (consumerEpoch < reqEpoch) { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 9be94eb5b6..3af8901b2b 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -238,9 +238,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (msgType == TDMT_VND_SUBMIT) { if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; - if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { - // TODO handle sma error - } void* data = taosMemoryMalloc(msgLen); if (data == NULL) { return -1; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2ecaeff747..0c38d6442b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -67,7 +67,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; pReadHandle->cachedSchemaVer = -1; - pReadHandle->cachedSchemaUid = -1; + pReadHandle->cachedSchemaSuid = -1; pReadHandle->pSchema = NULL; pReadHandle->pSchemaWrapper = NULL; pReadHandle->tbIdHash = NULL; @@ -130,7 +130,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p // TODO set to real sversion /*int32_t sversion = 1;*/ int32_t sversion = htonl(pHandle->pBlock->sversion); - if (pHandle->cachedSchemaVer != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { + if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || + pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); if (pHandle->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", @@ -150,7 +151,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p return -1; } pHandle->cachedSchemaVer = sversion; - pHandle->cachedSchemaUid = pHandle->msgIter.suid; + pHandle->cachedSchemaSuid = pHandle->msgIter.suid; } STSchema* pTschema = pHandle->pSchema; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 17b8afda4b..90093f2510 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -150,7 +150,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { return code; _err: - tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ffbef4e765..d8bc87d471 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -176,13 +176,13 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->nDelOp++; - tsdbError("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 + tsdbError("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; _err: - tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 + tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index da3c8e7c93..ab2efa4791 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -225,6 +225,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0); @@ -236,13 +237,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0); - case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); - case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); - case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: @@ -279,7 +277,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - // blockDebugShowData(data); + // blockDebugShowData(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } @@ -305,6 +303,10 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pVnode->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + static int64_t vndTick = 0; + if (++vndTick % 10 == 1) { + vTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); + } syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); @@ -791,7 +793,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq msgIter.suid = 0; } +#ifdef TD_DEBUG_PRINT_ROW vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); +#endif tDecoderClear(&decoder); } else { submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN); @@ -902,4 +906,4 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo // 2. adjust hash range / compact / remove wals / rename vgroups // 3. reload sync return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 087dcac7b7..b4021ec73d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -180,10 +180,18 @@ static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } -static void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { +static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbMeta) { SVnode *pVnode = pFsm->data; vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode)); +#if 0 +// send response + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); +#endif + // todo rpc response here // build rpc msg // put into apply queue @@ -213,6 +221,7 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); + } else { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 3b8c94e413..0161fac9b5 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -373,7 +373,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); EXPECT_NE(pTsdb->pTfs, nullptr); - // generate SSubmitReq msg and update expired window + // generate SSubmitReq msg and update expire window int16_t schemaVer = 0; uint32_t mockRowLen = sizeof(STSRow); uint32_t mockRowNum = 2; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 502949f191..de38e3cb77 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -13,23 +13,22 @@ * along with this program. If not, see . */ -#include "trpc.h" -#include "query.h" -#include "tname.h" #include "catalogInt.h" +#include "query.h" #include "systable.h" +#include "tname.h" #include "tref.h" +#include "trpc.h" SCatalogMgmt gCtgMgmt = {0}; - int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { - int32_t code = 0; - STableMeta *tblMeta = NULL; + int32_t code = 0; + STableMeta* tblMeta = NULL; SCtgTbMetaCtx tbCtx = {0}; tbCtx.flag = CTG_FLAG_UNKNOWN_STB; tbCtx.pName = pTableName; - + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta)); if (NULL == tblMeta) { @@ -39,13 +38,13 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - + if (TSDB_SUPER_TABLE == tblMeta->tableType) { CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); } else { CTG_ERR_JRET(ctgDropTbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq)); } - + _return: taosMemoryFreeClear(tblMeta); @@ -53,7 +52,8 @@ _return: CTG_RET(code); } -int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, + SDBVgInfo** pInfo) { int32_t code = 0; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache)); @@ -62,7 +62,7 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, con return TSDB_CODE_SUCCESS; } - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); @@ -80,17 +80,17 @@ _return: taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; - + CTG_RET(code); } -int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { - int32_t code = 0; +int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* dbFName) { + int32_t code = 0; SCtgDBCache* dbCache = NULL; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); @@ -100,7 +100,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgReleaseVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } - + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.numOfTable = 0; @@ -119,18 +119,16 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, return TSDB_CODE_SUCCESS; } - - -int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOutput, bool syncReq) { +int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput** pOutput, bool syncReq) { SVgroupInfo vgroupInfo = {0}; - int32_t code = 0; + int32_t code = 0; if (!CTG_FLAG_IS_SYS_DB(ctx->flag)) { CTG_ERR_RET(catalogGetTableHashVgroup(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo)); } STableMetaOutput moutput = {0}; - STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); + STableMetaOutput* output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -139,7 +137,8 @@ int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOut if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); - CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, output, NULL)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char*)ctx->pName->dbname, (char*)ctx->pName->tname, + output, NULL)); } else if (CTG_FLAG_IS_STB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); @@ -150,7 +149,8 @@ int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOut CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), + ctx->flag); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL)); @@ -159,7 +159,7 @@ int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOut ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); taosMemoryFreeClear(output->tbMeta); - + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, output, NULL)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; @@ -173,14 +173,14 @@ int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOut if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); } - + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { taosMemoryFreeClear(output->tbMeta); - - SET_META_TYPE_CTABLE(output->metaType); + + SET_META_TYPE_CTABLE(output->metaType); } } } @@ -192,9 +192,11 @@ int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOut } if (CTG_IS_META_TABLE(output->metaType)) { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, + output->tbMeta->tableType); } else { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, + output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); } if (pOutput) { @@ -209,7 +211,7 @@ _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_RET(code); } @@ -221,7 +223,8 @@ int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTabl CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, ctx, pTableMeta)); if (*pTableMeta) { - if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) { + if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) { return TSDB_CODE_SUCCESS; } @@ -231,14 +234,13 @@ int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTabl if (CTG_FLAG_IS_UNKNOWN_STB(ctx->flag)) { CTG_FLAG_SET_STB(ctx->flag, ctx->tbInfo.tbType); } - + return TSDB_CODE_SUCCESS; } - int32_t ctgGetTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { - int32_t code = 0; - STableMetaOutput *output = NULL; + int32_t code = 0; + STableMetaOutput* output = NULL; CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), ctx, pTableMeta)); if (*pTableMeta) { @@ -255,7 +257,7 @@ int32_t ctgGetTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { if (CTG_IS_META_BOTH(output->metaType)) { memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - + *pTableMeta = output->tbMeta; goto _return; } @@ -275,7 +277,7 @@ int32_t ctgGetTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = ctx->flag; stbCtx.pName = &stbName; - + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, pTableMeta)); if (NULL == *pTableMeta) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, ctx->pName->tname); @@ -314,43 +316,43 @@ _return: CTG_RET(code); } -int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp *rspMsg, bool syncOp) { - STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); +int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { + STableMetaOutput* output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - + int32_t code = 0; strcpy(output->dbFName, rspMsg->dbFName); strcpy(output->tbName, rspMsg->tbName); output->dbId = rspMsg->dbId; - + SET_META_TYPE_TABLE(output->metaType); - + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, rspMsg->tableType == TSDB_SUPER_TABLE, &output->tbMeta)); CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncOp)); return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_RET(code); } - -int32_t ctgChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { - bool inCache = false; +int32_t ctgChkAuth(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, + AUTH_TYPE type, bool* pass) { + bool inCache = false; int32_t code = 0; - + *pass = false; - + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); if (inCache) { @@ -359,7 +361,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const c SGetUserAuthRsp authRsp = {0}; CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), user, &authRsp, NULL)); - + if (authRsp.superAuth) { *pass = true; goto _return; @@ -383,25 +385,25 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SName* pTableName, SArray** pVgList) { - STableMeta *tbMeta = NULL; - int32_t code = 0; - SVgroupInfo vgroupInfo = {0}; - SCtgDBCache* dbCache = NULL; - SArray *vgList = NULL; - SDBVgInfo *vgInfo = NULL; +int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, SName* pTableName, SArray** pVgList) { + STableMeta* tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SCtgDBCache* dbCache = NULL; + SArray* vgList = NULL; + SDBVgInfo* vgInfo = NULL; SCtgTbMetaCtx ctx = {0}; ctx.pName = pTableName; ctx.flag = CTG_FLAG_UNKNOWN_STB; *pVgList = NULL; - + CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &tbMeta)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - SHashObj *vgHash = NULL; + SHashObj* vgHash = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, &dbCache, &vgInfo)); if (dbCache) { @@ -416,7 +418,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, // USE HASH METHOD INSTEAD OF VGID IN TBMETA ctgError("invalid method to get none stb vgInfo, tbType:%d", tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); - + #if 0 int32_t vgId = tbMeta->vgId; if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { @@ -437,7 +439,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, *pVgList = vgList; vgList = NULL; -#endif +#endif } _return: @@ -462,7 +464,7 @@ _return: CTG_RET(code); } -int32_t catalogInit(SCatalogCfg *cfg) { +int32_t catalogInit(SCatalogCfg* cfg) { if (gCtgMgmt.pCluster) { qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); @@ -495,7 +497,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), + false, HASH_ENTRY_LOCK); if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -505,7 +508,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); } - + gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); @@ -521,7 +524,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(ctgStartUpdateThread()); - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, + gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } @@ -532,19 +536,19 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { } if (NULL == gCtgMgmt.pCluster) { - qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%" PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } - int32_t code = 0; - SCatalog *clusterCtg = NULL; + int32_t code = 0; + SCatalog* clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); + SCatalog** ctg = (SCatalog**)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; - qDebug("got catalog handle from cache, clusterId:%"PRIx64", CTG:%p", clusterId, *ctg); + qDebug("got catalog handle from cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, *ctg); return TSDB_CODE_SUCCESS; } @@ -559,7 +563,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_ENTRY_LOCK); if (NULL == clusterCtg->dbCache) { qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -571,12 +576,12 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { ctgFreeHandle(clusterCtg); continue; } - - qError("taosHashPut CTG to cache failed, clusterId:%"PRIx64, clusterId); + + qError("taosHashPut CTG to cache failed, clusterId:%" PRIx64, clusterId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("add CTG to cache, clusterId:%"PRIx64", CTG:%p", clusterId, clusterCtg); + qDebug("add CTG to cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, clusterCtg); break; } @@ -584,13 +589,13 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { *catalogHandle = clusterCtg; CTG_CACHE_STAT_INC(clusterNum, 1); - + return TSDB_CODE_SUCCESS; _return: ctgFreeHandle(clusterCtg); - + CTG_RET(code); } @@ -600,28 +605,28 @@ void catalogFreeHandle(SCatalog* pCtg) { } if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%" PRIx64, pCtg->clusterId); return; } CTG_CACHE_STAT_DEC(clusterNum, 1); uint64_t clusterId = pCtg->clusterId; - + ctgFreeHandle(pCtg); - - ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); + + ctgInfo("handle freed, culsterId:%" PRIx64, clusterId); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache *dbCache = NULL; - int32_t code = 0; + SCtgDBCache* dbCache = NULL; + int32_t code = 0; CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { @@ -645,7 +650,8 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* dbFName, + SArray** vgroupList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == pTrans || NULL == pMgmtEps || NULL == vgroupList) { @@ -653,10 +659,10 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } SCtgDBCache* dbCache = NULL; - int32_t code = 0; - SArray *vgList = NULL; - SHashObj *vgHash = NULL; - SDBVgInfo *vgInfo = NULL; + int32_t code = 0; + SArray* vgList = NULL; + SHashObj* vgHash = NULL; + SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; @@ -681,15 +687,14 @@ _return: taosMemoryFreeClear(vgInfo); } - CTG_API_LEAVE(code); + CTG_API_LEAVE(code); } - int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { ctgFreeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); @@ -702,12 +707,11 @@ _return: CTG_API_LEAVE(code); } - int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -719,17 +723,17 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_ERR_JRET(ctgDropDbCacheEnqueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) { +int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == epSet) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -737,7 +741,7 @@ int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, CTG_ERR_JRET(ctgUpdateVgEpsetEnqueue(pCtg, (char*)dbFName, vgId, epSet)); _return: - + CTG_API_LEAVE(code); } @@ -745,7 +749,7 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -757,16 +761,15 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, pTableName, true)); _return: - + CTG_API_LEAVE(code); } - int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -778,23 +781,25 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + STableMeta** pTableMeta) { CTG_API_ENTER(); SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)pTableName; ctx.flag = CTG_FLAG_UNKNOWN_STB; - + CTG_API_LEAVE(ctgGetTbMeta(pCtg, pTrans, pMgmtEps, &ctx, pTableMeta)); } -int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + STableMeta** pTableMeta) { CTG_API_ENTER(); SCtgTbMetaCtx ctx = {0}; @@ -804,7 +809,7 @@ int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtE CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } -int32_t catalogUpdateTableMeta(SCatalog* pCtg, STableMetaRsp *pMsg) { +int32_t catalogUpdateTableMeta(SCatalog* pCtg, STableMetaRsp* pMsg) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pMsg) { @@ -813,20 +818,20 @@ int32_t catalogUpdateTableMeta(SCatalog* pCtg, STableMetaRsp *pMsg) { int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbMeta(pCtg, pMsg, true)); - + _return: - + CTG_API_LEAVE(code); } -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, SArray* pTables) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SName name; + SName name; int32_t sver = 0; int32_t tver = 0; int32_t tbNum = taosArrayGetSize(pTables); @@ -835,7 +840,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { continue; } - + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (CTG_IS_SYS_DBNAME(name.dbname)) { @@ -868,8 +873,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - -int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { +int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* dbFName) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) { @@ -879,7 +883,8 @@ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmt CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName)); } -int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + int32_t isSTable) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { @@ -893,7 +898,8 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(ctgRefreshTbMeta(CTG_PARAMS_LIST(), &ctx, NULL, true)); } -int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); SCtgTbMetaCtx ctx = {0}; @@ -903,7 +909,8 @@ int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* p CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } -int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { +int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + SArray** pVgList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { @@ -918,8 +925,8 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pM CTG_API_LEAVE(ctgGetTbDistVgInfo(pCtg, pTrans, pMgmtEps, (SName*)pTableName, pVgList)); } - -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t catalogGetTableHashVgroup(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + SVgroupInfo* pVgroup) { CTG_API_ENTER(); if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { @@ -928,11 +935,11 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pM } SCtgDBCache* dbCache = NULL; - int32_t code = 0; - char db[TSDB_DB_FNAME_LEN] = {0}; + int32_t code = 0; + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - SDBVgInfo *vgInfo = NULL; + SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); @@ -952,8 +959,8 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { +int32_t catalogGetAllMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, + SMetaData* pRsp) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { @@ -975,14 +982,14 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgError("taosArrayInit %d failed", tbNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + for (int32_t i = 0; i < tbNum; ++i) { - SName *name = taosArrayGet(pReq->pTableMeta, i); - STableMeta *pTableMeta = NULL; + SName* name = taosArrayGet(pReq->pTableMeta, i); + STableMeta* pTableMeta = NULL; SCtgTbMetaCtx ctx = {0}; ctx.pName = name; ctx.flag = CTG_FLAG_UNKNOWN_STB; - + CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &pTableMeta)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { @@ -1000,31 +1007,32 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_API_LEAVE(TSDB_CODE_SUCCESS); -_return: +_return: if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { - STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); + STableMeta* pMeta = taosArrayGetP(pRsp->pTableMeta, i); taosMemoryFreeClear(pMeta); } - + taosArrayDestroy(pRsp->pTableMeta); pRsp->pTableMeta = NULL; } - + CTG_API_LEAVE(code); } -int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) { +int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, uint64_t reqId, + const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == fp || NULL == param) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t code = 0, taskNum = 0; - SCtgJob *pJob = NULL; + int32_t code = 0, taskNum = 0; + SCtgJob* pJob = NULL; CTG_ERR_JRET(ctgInitJob(CTG_PARAMS_LIST(), &pJob, reqId, pReq, fp, param, &taskNum)); if (taskNum <= 0) { SMetaData* pMetaData = taosMemoryCalloc(1, sizeof(SMetaData)); @@ -1035,8 +1043,8 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmt CTG_ERR_JRET(ctgLaunchJob(pJob)); // NOTE: here the assignment of jobId is invalid, may over-write the true scheduler created query job. -// *jobId = pJob->refId; - + // *jobId = pJob->refId; + _return: if (pJob) { taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); @@ -1045,15 +1053,15 @@ _return: taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); } } - + CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pQnodeList) { +int32_t catalogGetQnodeList(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, SArray* pQnodeList) { CTG_API_ENTER(); - + int32_t code = 0; - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pQnodeList) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pQnodeList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1064,29 +1072,29 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion** stables, uint32_t* num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion** dbs, uint32_t* num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == dbs || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void**)dbs, num, sizeof(SDbVgVersion))); } -int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == users || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1100,11 +1108,11 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ } } - uint32_t i = 0; - SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); + uint32_t i = 0; + SCtgUserAuth* pAuth = taosHashIterate(pCtg->userCache, NULL); while (pAuth != NULL) { size_t len = 0; - void *key = taosHashGetKey(pAuth, &len); + void* key = taosHashGetKey(pAuth, &len); strncpy((*users)[i].user, key, len); (*users)[i].user[len] = 0; (*users)[i].version = pAuth->version; @@ -1115,10 +1123,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - -int32_t catalogGetDBCfg(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { +int32_t catalogGetDBCfg(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1126,9 +1133,10 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, co CTG_API_LEAVE(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), dbFName, pDbCfg, NULL)); } -int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { +int32_t catalogGetIndexMeta(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* indexName, + SIndexInfo* pInfo) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1136,9 +1144,10 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL)); } -int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pRes) { +int32_t catalogGetTableIndex(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pTableName, + SArray** pRes) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pRes) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1146,32 +1155,33 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEp CTG_API_LEAVE(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), (SName*)pTableName, pRes, NULL)); } - -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo) { +int32_t catalogGetUdfInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* funcName, + SFuncInfo* pInfo) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; CTG_ERR_JRET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), funcName, pInfo, NULL)); - + _return: - + CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { +int32_t catalogChkAuth(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, + AUTH_TYPE type, bool* pass) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; CTG_ERR_JRET(ctgChkAuth(CTG_PARAMS_LIST(), user, dbFName, type, pass)); - + _return: CTG_API_LEAVE(code); @@ -1187,10 +1197,9 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_LEAVE(ctgUpdateUserEnqueue(pCtg, pAuth, false)); } - void catalogDestroy(void) { qInfo("start to destroy catalog"); - + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { return; } @@ -1204,21 +1213,21 @@ void catalogDestroy(void) { while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); } - + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); - SCatalog *pCtg = NULL; - void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + SCatalog* pCtg = NULL; + void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { - pCtg = *(SCatalog **)pIter; + pCtg = *(SCatalog**)pIter; if (pCtg) { catalogFreeHandle(pCtg); } - + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - + taosHashCleanup(gCtgMgmt.pCluster); gCtgMgmt.pCluster = NULL; @@ -1226,6 +1235,3 @@ void catalogDestroy(void) { qInfo("catalog destroyed"); } - - - diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 2574528d15..f61a3637ed 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -525,7 +525,7 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + taosArrayPush(pJob->jobRes.pTableIndex, &res); return TSDB_CODE_SUCCESS; } @@ -875,7 +875,9 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf TSWAP(pTask->res, pTask->msgCtx.out); _return: - + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } ctgHandleTaskEnd(pTask, code); CTG_RET(code); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a8a95b513a..265f968d24 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -458,7 +458,6 @@ typedef struct STimeWindowSupp { int64_t waterMark; TSKEY maxTs; SColumnInfoData timeWindowData; // query time window info for scalar function execution. - SHashObj *winMap; } STimeWindowAggSupp; typedef struct SIntervalAggOperatorInfo { @@ -760,7 +759,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); @@ -809,6 +808,11 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream); + +SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + SExecTaskInfo* pTaskInfo); + SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, @@ -903,10 +907,11 @@ SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); -int64_t getSmaWaterMark(int64_t interval, double filesFactor); -bool isSmaStream(int8_t triggerType); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); +int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, + SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f50c7cdae5..5a68a9d7e1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1819,9 +1819,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } } -static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree); +static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { if (pFilterNode == NULL) { return; } @@ -1840,30 +1840,29 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) { bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); - extractQualifiedTupleByFilterResult(pBlock, rowRes, keep, needFree); + extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); blockDataUpdateTsWindow(pBlock, 0); } -void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree) { +void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { if (keep) { return; } if (rowRes != NULL) { - SSDataBlock* px = createOneDataBlock(pBlock, false); - blockDataEnsureCapacity(px, pBlock->info.rows); + SSDataBlock* px = createOneDataBlock(pBlock, true); int32_t totalRows = pBlock->info.rows; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); - + SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. - if (pSrc->pData == NULL) { + if (pDst->pData == NULL) { continue; } + colInfoDataCleanup(pDst, pBlock->info.rows); + int32_t numOfRows = 0; for (int32_t j = 0; j < totalRows; ++j) { if (rowRes[j] == 0) { @@ -1883,20 +1882,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR } else { ASSERT(pBlock->info.rows == numOfRows); } - - SColumnInfoData tmp = *pSrc; - *pSrc = *pDst; - *pDst = tmp; - - if (!needFree) { - if (IS_VAR_DATA_TYPE(pDst->info.type)) { // this elements do not need free - pDst->varmeta.offset = NULL; - } else { - pDst->nullbitmap = NULL; - } - pDst->pData = NULL; - } } + blockDataDestroy(px); // fix memory leak } else { // do nothing @@ -1956,6 +1943,57 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ } } +int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, + const int32_t* rowCellOffset, SSDataBlock* pBlock, + SExecTaskInfo* pTaskInfo) { + SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); + + doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + if (pRow->numOfRows == 0) { + releaseBufPage(pBuf, page); + return 0; + } + + while (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + int32_t code = blockDataEnsureCapacity(pBlock, pBlock->info.capacity * 1.25); + if (TAOS_FAILED(code)) { + releaseBufPage(pBuf, page); + qError("%s ensure result data capacity failed, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + longjmp(pTaskInfo->env, code); + } + } + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + longjmp(pTaskInfo->env, code); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstartts, top(k, 20) + // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + + releaseBufPage(pBuf, page); + pBlock->info.rows += pRow->numOfRows; + + return 0; +} + int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { @@ -2086,11 +2124,6 @@ static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, } int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t capacity) { - // for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - // SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); - // p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); - // } - int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, pBlock, capacity - pBlock->info.rows); pBlock->info.rows += numOfRows; @@ -2845,7 +2878,6 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { return seqLoadRemoteData(pOperator); } else { return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - // return concurrentlyLoadRemoteData(pOperator); } } @@ -2871,9 +2903,14 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo) { +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); + if (numOfSources == 0) { + qError("%s invalid number: %d of sources in exchange operator", id, (int32_t) numOfSources); + return TSDB_CODE_INVALID_PARA; + } + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { @@ -2895,7 +2932,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode goto _error; } - int32_t code = initExchangeOperator(pExNode, pInfo); + int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2924,7 +2961,7 @@ _error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; return NULL; } @@ -3693,7 +3730,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - doFilter(pProjectInfo->pFilterNode, pBlock, true); + doFilter(pProjectInfo->pFilterNode, pBlock); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); @@ -4671,24 +4708,28 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, - .winMap = NULL, }; - if (isSmaStream(pIntervalPhyNode->window.triggerType)) { - if (FLT_LESS(pIntervalPhyNode->window.filesFactor, 1.000000)) { - as.calTrigger = STREAM_TRIGGER_AT_ONCE_SMA; - } else { - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); - as.winMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK); - as.waterMark = getSmaWaterMark(interval.interval, pIntervalPhyNode->window.filesFactor); - as.calTrigger = STREAM_TRIGGER_WINDOW_CLOSE_SMA; - } - } int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { + SMergeIntervalPhysiNode * pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; + + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + + SInterval interval = {.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; + + int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { qDebug("[******]create Semi"); int32_t children = 0; @@ -5208,7 +5249,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond); if (NULL == (*pTaskInfo)->pRoot) { - code = terrno; + code = (*pTaskInfo)->code; goto _complete; } @@ -5221,7 +5262,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead _complete: taosMemoryFreeClear(*pTaskInfo); - terrno = code; return code; } @@ -5380,16 +5420,3 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF return code; } -int64_t getSmaWaterMark(int64_t interval, double filesFactor) { - int64_t waterMark = 0; - ASSERT(FLT_GREATEREQUAL(filesFactor, 0.000000)); - waterMark = -1 * filesFactor; - return waterMark; -} - -bool isSmaStream(int8_t triggerType) { - if (triggerType == STREAM_TRIGGER_AT_ONCE || triggerType == STREAM_TRIGGER_WINDOW_CLOSE) { - return false; - } - return true; -} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 5965f5d7ad..132f93a6a5 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { while(1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes, true); + doFilter(pInfo->pCondition, pRes); bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6387802303..b5f8ad0ae2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, false); + doFilter(pTableScanInfo->pFilterNode, pBlock); int64_t et = taosGetTimestampMs(); pTableScanInfo->readRecorder.filterTime += (et - st); @@ -948,7 +948,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); } - doFilter(pInfo->pCondition, pInfo->pRes, false); + doFilter(pInfo->pCondition, pInfo->pRes); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); break; } @@ -1028,10 +1028,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan goto _error; } - if (isSmaStream(pTableScanNode->triggerType)) { - pTwSup->waterMark = getSmaWaterMark(pSTInfo->interval.interval, pTableScanNode->filesFactor); - } - if (pSTInfo->interval.interval > 0 && pDataReader) { pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark); } else { @@ -1720,7 +1716,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } pRes->info.rows = count; - doFilter(pInfo->pFilterNode, pRes, true); + doFilter(pInfo->pFilterNode, pRes); pOperator->resultInfo.totalRows += pRes->info.rows; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index dcaa95e28a..ff093741fa 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -296,7 +296,10 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { } SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, - SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) { + SArray* pColMatchInfo, SOperatorInfo* pOperator) { + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + blockDataCleanup(pDataBlock); SSDataBlock* p = tsortGetSortedDataBlock(pHandle); @@ -354,6 +357,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } blockDataDestroy(p); + + qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } @@ -371,7 +376,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { } SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, - pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pInfo); + pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d981b5034c..e70a4c413c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -750,14 +750,15 @@ int64_t getReskey(void* data, int32_t index) { return *(int64_t*)pos->key; } -static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated) { +static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, + SArray* pUpdated) { int32_t size = taosArrayGetSize(pUpdated); - int32_t index = binarySearch(pUpdated, size, result->win.skey, TSDB_ORDER_DESC, getReskey); + int32_t index = binarySearch(pUpdated, size, ts, TSDB_ORDER_DESC, getReskey); if (index == -1) { index = 0; } else { TSKEY resTs = getReskey(pUpdated, index); - if (resTs < result->win.skey) { + if (resTs < ts) { index++; } else { return TSDB_CODE_SUCCESS; @@ -769,14 +770,18 @@ static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated return TSDB_CODE_OUT_OF_MEMORY; } newPos->groupId = groupId; - newPos->pos = (SResultRowPosition){.pageId = result->pageId, .offset = result->offset}; - *(int64_t*)newPos->key = result->win.skey; + newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset}; + *(int64_t*)newPos->key = ts; if (taosArrayInsert(pUpdated, index, &newPos) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; } +static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { + return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated); +} + static void removeResult(SArray* pUpdated, TSKEY key) { int32_t size = taosArrayGetSize(pUpdated); int32_t index = binarySearch(pUpdated, size, key, TSDB_ORDER_DESC, getReskey); @@ -818,12 +823,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE_SMA) { - saveResult(pResult, tableGroupId, pUpdated); - } - if (pInfo->twAggSup.winMap) { - taosHashRemove(pInfo->twAggSup.winMap, &win.skey, sizeof(TSKEY)); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResultRow(pResult, tableGroupId, pUpdated); } } @@ -872,12 +873,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE_SMA) { - saveResult(pResult, tableGroupId, pUpdated); - } - if (pInfo->twAggSup.winMap) { - taosHashRemove(pInfo->twAggSup.winMap, &win.skey, sizeof(TSKEY)); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResultRow(pResult, tableGroupId, pUpdated); } } @@ -1251,6 +1248,23 @@ static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, SInterva } } +static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { + void* pIte = NULL; + size_t keyLen = 0; + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + uint64_t groupId = *(uint64_t*)key; + ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); + TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); + SResultRowPosition* pPos = (SResultRowPosition*)pIte; + int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, SArray* closeWins) { void* pIte = NULL; @@ -1264,28 +1278,16 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); if (win.ekey < pSup->maxTs - pSup->waterMark) { - if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE_SMA) { - if (taosHashGet(pSup->winMap, &win.skey, sizeof(TSKEY))) { - continue; - } - } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - if (pSup->calTrigger != STREAM_TRIGGER_AT_ONCE_SMA && pSup->calTrigger != STREAM_TRIGGER_WINDOW_CLOSE_SMA) { - taosHashRemove(pHashMap, keyBuf, keyLen); + taosHashRemove(pHashMap, keyBuf, keyLen); + SResultRowPosition* pPos = (SResultRowPosition*)pIte; + if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (pos == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pos->groupId = groupId; - pos->pos = *(SResultRowPosition*)pIte; - *(int64_t*)pos->key = ts; - if (!taosArrayPush(closeWins, &pos)) { - taosMemoryFree(pos); - return TSDB_CODE_OUT_OF_MEMORY; - } - taosHashPut(pSup->winMap, &win.skey, sizeof(TSKEY), NULL, 0); } } return TSDB_CODE_SUCCESS; @@ -1312,8 +1314,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - SArray* pClosed = taosArrayInit(4, POINTER_BYTES); - while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -1332,22 +1332,18 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; + } else if (pBlock->info.type == STREAM_GET_ALL && + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + continue; } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pClosed); - finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, pInfo->binfo.rowCellInfoOffset); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE_SMA) { - taosArrayAddAll(pUpdated, pClosed); - } - - taosArrayDestroy(pClosed); finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); @@ -1881,8 +1877,8 @@ _error: return NULL; } -static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId, SArray* pUpdated) { +static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId, + SArray* pUpdated) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -1897,7 +1893,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; } else { - return ; + return; } int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); @@ -1914,13 +1910,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc pos->groupId = tableGroupId; pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; *(int64_t*)pos->key = pResult->win.skey; - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, - nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { - saveResult(pResult, tableGroupId, pUpdated); + saveResultRow(pResult, tableGroupId, pUpdated); } // window start(end) key interpolation - // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); + // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); @@ -2011,7 +2008,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - SArray* pClosed = taosArrayInit(4, POINTER_BYTES); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2049,10 +2045,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; - doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, - pChildInfo->primaryTsIndex, pChildOp->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, - pOperator->numOfExprs, pOperator->pTaskInfo); + doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, pChildInfo->primaryTsIndex, + pChildOp->numOfExprs, pBlock, NULL); + rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, + pOperator->pTaskInfo); taosArrayDestroy(pUpWins); continue; } @@ -2060,9 +2056,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); taosArrayDestroy(pUpWins); break; + } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo) && + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + continue; } + if (isFinalInterval(pInfo)) { - int32_t chIndex = getChildIndex(pBlock); + int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); // if chIndex + 1 - size > 0, add new child for (int32_t i = 0; i < chIndex + 1 - size; i++) { @@ -2072,7 +2073,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } taosArrayPush(pInfo->pChildren, &pChildOp); } - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChInfo->binfo.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); @@ -2080,17 +2081,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } - + if (isFinalInterval(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pClosed); - finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, - pInfo->binfo.rowCellInfoOffset); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - taosArrayAddAll(pUpdated, pClosed); - } + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); } - taosArrayDestroy(pClosed); finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); @@ -2109,31 +2103,31 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { - SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild) { + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = (SInterval) {.interval = pIntervalPhyNode->interval, - .sliding = pIntervalPhyNode->sliding, - .intervalUnit = pIntervalPhyNode->intervalUnit, - .slidingUnit = pIntervalPhyNode->slidingUnit, - .offset = pIntervalPhyNode->offset, - .precision = - ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, + pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, - .winMap = NULL, }; + }; pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); @@ -2160,7 +2154,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (!isFinalInterval(pInfo)) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } - pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);\ + pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); pInfo->pUpdateRes->info.type = STREAM_REPROCESS; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pInfo->pPhyNode = nodesCloneNode(pPhyNode); @@ -2174,9 +2168,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, - destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, - NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2216,8 +2210,7 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { } } -int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResultBlock) { +int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) { pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pRes = pResultBlock; for (int32_t i = 0; i < numOfCols; ++i) { @@ -2656,8 +2649,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*); SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; } SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } -int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pClosed, int8_t calTrigger, - __get_win_info_ fn) { +int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pClosed, + __get_win_info_ fn) { // Todo(liuyao) save window to tdb int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { @@ -2665,19 +2658,9 @@ int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pC SResultWindowInfo* pSeWin = fn(pWin); if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) { if (!pSeWin->isClosed) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (pos == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pos->groupId = 0; - pos->pos = pSeWin->pos; - *(int64_t*)pos->key = pSeWin->win.ekey; - if (!taosArrayPush(pClosed, &pos)) { - taosMemoryFree(pos); - return TSDB_CODE_OUT_OF_MEMORY; - } pSeWin->isClosed = true; - if (calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); pSeWin->isOutput = true; } } @@ -2688,6 +2671,19 @@ int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pC return TSDB_CODE_SUCCESS; } +int32_t getAllSessionWindow(SArray* pWins, SArray* pClosed, __get_win_info_ fn) { + int32_t size = taosArrayGetSize(pWins); + for (int32_t i = 0; i < size; i++) { + void* pWin = taosArrayGet(pWins, i); + SResultWindowInfo* pSeWin = fn(pWin); + if (!pSeWin->isClosed) { + int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); + pSeWin->isOutput = true; + } + } + return TSDB_CODE_SUCCESS; +} + static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2710,6 +2706,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -2730,7 +2727,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } taosArrayDestroy(pWins); continue; + } else if (pBlock->info.type == STREAM_GET_ALL && + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getSessionWinInfo); + continue; } + if (isFinalSession(pInfo)) { int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); @@ -2742,15 +2744,10 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - SArray* pClosed = taosArrayInit(16, POINTER_BYTES); - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, pInfo->twAggSup.calTrigger, + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getSessionWinInfo); - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); copyUpdateResult(pStUpdated, pUpdated, pBInfo->pRes->info.groupId); taosHashCleanup(pStUpdated); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - taosArrayAddAll(pUpdated, pClosed); - } finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); @@ -3074,6 +3071,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3085,6 +3083,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; + } else if (pBlock->info.type == STREAM_GET_ALL && + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getStateWinInfo); + continue; } doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); @@ -3092,15 +3094,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - SArray* pClosed = taosArrayInit(16, POINTER_BYTES); - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, pInfo->twAggSup.calTrigger, + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getStateWinInfo); - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); copyUpdateResult(pSeUpdated, pUpdated, pBInfo->pRes->info.groupId); taosHashCleanup(pSeUpdated); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - taosArrayAddAll(pUpdated, pClosed); - } finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); @@ -3142,7 +3139,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys .waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType, .maxTs = INT64_MIN, - .winMap = NULL, }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); @@ -3195,3 +3191,281 @@ _error: pTaskInfo->code = code; return NULL; } + +typedef struct SMergeIntervalAggOperatorInfo { + SIntervalAggOperatorInfo intervalAggOperatorInfo; + + SHashObj* groupIntervalHash; + bool hasGroupId; + uint64_t groupId; + SSDataBlock* prefetchedBlock; + bool inputBlocksFinished; +} SMergeIntervalAggOperatorInfo; + +void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; + taosHashCleanup(miaInfo->groupIntervalHash); + destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); +} + +static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, + STimeWindow* newWin) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + + STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + if (prevWin == NULL) { + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + return 0; + } + + if (newWin == NULL || (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey) ) { + SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + ASSERT(p1 != NULL); + + finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, iaInfo->binfo.pCtx, pOperatorInfo->pExpr, + pOperatorInfo->numOfExprs, iaInfo->binfo.rowCellInfoOffset, pResultBlock, + pTaskInfo); + taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + if (newWin == NULL) { + taosHashRemove(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + } else { + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + } + } + + return 0; +} + +static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag, SSDataBlock* pResultBlock) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + int64_t* tsCols = extractTsCol(pBlock, iaInfo); + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + TSKEY blockStartTs = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; + + STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, + iaInfo->interval.precision, &iaInfo->win); + + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx, + numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order); + ASSERT(forwardRows > 0); + + // prev time window not interpolation yet. + if (iaInfo->timeWindowInterpo) { + SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult); + doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); + + // restore current time window + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + iaInfo->binfo.pCtx, numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // window start key interpolation + doWindowBorderInterpolation(iaInfo, pBlock, numOfOutput, iaInfo->binfo.pCtx, pResult, &win, startPos, forwardRows); + } + + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &win, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, iaInfo->order); + doCloseWindow(pResultRowInfo, iaInfo, pResult); + + // output previous interval results after this interval (&win) is closed + outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = forwardRows - 1 + startPos; + startPos = getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->order); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + iaInfo->binfo.pCtx, numOfOutput, iaInfo->binfo.rowCellInfoOffset, + &iaInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order); + + // window start(end) key interpolation + doWindowBorderInterpolation(iaInfo, pBlock, numOfOutput, iaInfo->binfo.pCtx, pResult, &nextWin, startPos, + forwardRows); + + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &nextWin, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); + doCloseWindow(pResultRowInfo, iaInfo, pResult); + + // output previous interval results after this interval (&nextWin) is closed + outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin); + } + + if (iaInfo->timeWindowInterpo) { + saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols); + } +} + +static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pRes = iaInfo->binfo.pRes; + blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); + + if (!miaInfo->inputBlocksFinished) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; + while (1) { + SSDataBlock* pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->groupId = pBlock->info.groupId; + } + + if (pBlock == NULL) { + miaInfo->inputBlocksFinished = true; + break; + } + + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + miaInfo->prefetchedBlock = pBlock; + break; + } + + getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); + setInputDataBlock(pOperator, iaInfo->binfo.pCtx, pBlock, iaInfo->order, scanFlag, true); + STableQueryInfo* pTableQueryInfo = iaInfo->pCurrent; + + setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); + doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + } + + pRes->info.groupId = miaInfo->groupId; + } else { + void* p = taosHashIterate(miaInfo->groupIntervalHash, NULL); + if (p != NULL) { + size_t len = 0; + uint64_t* pKey = taosHashGetKey(p, &len); + outputPrevIntervalResult(pOperator, *pKey, pRes, NULL); + } + } + + if (pRes->info.rows == 0) { + doSetOperatorCompleted(pOperator); + } + + size_t rows = pRes->info.rows; + pOperator->resultInfo.totalRows += rows; + return (rows == 0) ? NULL : pRes; +} + +SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + SExecTaskInfo* pTaskInfo) { + SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (miaInfo == NULL || pOperator == NULL) { + goto _error; + } + + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + + iaInfo->win = pTaskInfo->window; + iaInfo->order = TSDB_ORDER_ASC; + iaInfo->interval = *pInterval; + + iaInfo->execModel = pTaskInfo->execModel; + + iaInfo->primaryTsIndex = primaryTsSlotId; + miaInfo->groupIntervalHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&iaInfo->binfo, &iaInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + + initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win); + + iaInfo->timeWindowInterpo = timeWindowinterpNeeded(iaInfo->binfo.pCtx, numOfCols, iaInfo); + if (iaInfo->timeWindowInterpo) { + iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); + } + + // iaInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || iaInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&iaInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "TimeMergeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = miaInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, + destroyMergeIntervalOperatorInfo, NULL, NULL, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyMergeIntervalOperatorInfo(miaInfo, numOfCols); + taosMemoryFreeClear(miaInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index ea8ded2c30..f63a9351c6 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -227,6 +227,8 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); + + // set current source id done if (pSource->src.pBlock == NULL) { pSource->src.rowIndex = -1; ++pHandle->numOfCompletedSources; @@ -426,8 +428,16 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages)); pHandle->totalElapsed = taosGetTimestampUs() - pHandle->startTs; - qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort elapsed:%"PRId64", total elapsed:%"PRId64, - pHandle->idStr, (int32_t) (sortPass + 1), pHandle->pBuf ? getTotalBufSize(pHandle->pBuf) : 0, pHandle->sortElapsed, pHandle->totalElapsed); + + if (sortPass > 0) { + size_t s = pHandle->pBuf ? getTotalBufSize(pHandle->pBuf) : 0; + qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%" PRIzu + ", sort elapsed:%" PRId64 ", total elapsed:%" PRId64, + pHandle->idStr, (int32_t)(sortPass + 1), s, pHandle->sortElapsed, pHandle->totalElapsed); + } else { + qDebug("%s ordered source:%"PRIzu", available buf:%d, no need internal sort", pHandle->idStr, numOfSources, + pHandle->numOfPages); + } int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize); blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index b623c77110..8274a33eb1 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -112,6 +112,7 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t getSpreadInfoSize(); +int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); @@ -120,6 +121,7 @@ int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx); int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t getElapsedInfoSize(); +int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); @@ -128,6 +130,7 @@ int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx); int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t getHistogramInfoSize(); +int32_t histogramCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t hllFunction(SqlFunctionCtx* pCtx); @@ -135,6 +138,7 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx); int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t getHLLInfoSize(); +int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index 29dd0bcd90..d1af6b6051 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -41,6 +41,7 @@ extern "C" { #define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) #define FUNC_MGT_SELECT_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(13) #define FUNC_MGT_REPEAT_SCAN_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(14) +#define FUNC_MGT_FORBID_FILL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(15) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a8e9cac65e..a1aa1a775c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -294,7 +294,8 @@ static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int pValue->notReserved = true; } - pFunc->node.resType = (SDataType){.bytes = getApercentileMaxSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getApercentileMaxSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -479,7 +480,8 @@ static int32_t translateElapsedImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t } } - pFunc->node.resType = (SDataType){.bytes = getElapsedInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getElapsedInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -593,7 +595,8 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32 return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -631,7 +634,8 @@ static int32_t translateHLLImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len } if (isPartial) { - pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; } @@ -1112,7 +1116,7 @@ static bool validateHourRange(int8_t hour) { } static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { - if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '-')) { + if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) { return true; } @@ -1127,7 +1131,7 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { char* tz = varDataVal(pVal->datum.p); int32_t len = varDataLen(pVal->datum.p); - char buf[3] = {0}; + char buf[3] = {0}; int8_t hour = -1, minute = -1; if (len == 0) { return false; @@ -1320,7 +1324,7 @@ static int32_t translateSelectValue(SFunctionNode* pFunc, char* pErrBuf, int32_t } static int32_t translateBlockDistFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - pFunc->node.resType = (SDataType) {.bytes = 128, .type = TSDB_DATA_TYPE_VARCHAR}; + pFunc->node.resType = (SDataType){.bytes = 128, .type = TSDB_DATA_TYPE_VARCHAR}; return TSDB_CODE_SUCCESS; } @@ -1329,7 +1333,6 @@ static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv return true; } - // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -1447,6 +1450,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = apercentileFunctionSetup, .processFunc = apercentileFunction, .finalizeFunc = apercentileFinalize, + .invertFunc = NULL, .combineFunc = apercentileCombine, .pPartialFunc = "_apercentile_partial", .pMergeFunc = "_apercentile_merge" @@ -1459,7 +1463,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getApercentileFuncEnv, .initFunc = apercentileFunctionSetup, .processFunc = apercentileFunction, - .finalizeFunc = apercentilePartialFinalize + .finalizeFunc = apercentilePartialFinalize, + .invertFunc = NULL, + .combineFunc = apercentileCombine, }, { .name = "_apercentile_merge", @@ -1469,7 +1475,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getApercentileFuncEnv, .initFunc = functionSetup, .processFunc = apercentileFunctionMerge, - .finalizeFunc = apercentileFinalize + .finalizeFunc = apercentileFinalize, + .invertFunc = NULL, + .combineFunc = apercentileCombine, }, { .name = "top", @@ -1503,6 +1511,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = spreadFunctionSetup, .processFunc = spreadFunction, .finalizeFunc = spreadFinalize, + .invertFunc = NULL, + .combineFunc = spreadCombine, .pPartialFunc = "_spread_partial", .pMergeFunc = "_spread_merge" }, @@ -1515,7 +1525,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSpreadFuncEnv, .initFunc = spreadFunctionSetup, .processFunc = spreadFunction, - .finalizeFunc = spreadPartialFinalize + .finalizeFunc = spreadPartialFinalize, + .invertFunc = NULL, + .combineFunc = spreadCombine, }, { .name = "_spread_merge", @@ -1526,7 +1538,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSpreadFuncEnv, .initFunc = spreadFunctionSetup, .processFunc = spreadFunctionMerge, - .finalizeFunc = spreadFinalize + .finalizeFunc = spreadFinalize, + .invertFunc = NULL, + .combineFunc = spreadCombine, }, { .name = "elapsed", @@ -1538,6 +1552,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunction, .finalizeFunc = elapsedFinalize, + .invertFunc = NULL, + .combineFunc = elapsedCombine, .pPartialFunc = "_elapsed_partial", .pMergeFunc = "_elapsed_merge" }, @@ -1550,7 +1566,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getElapsedFuncEnv, .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunction, - .finalizeFunc = elapsedPartialFinalize + .finalizeFunc = elapsedPartialFinalize, + .invertFunc = NULL, + .combineFunc = elapsedCombine, }, { .name = "_elapsed_merge", @@ -1561,7 +1579,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getElapsedFuncEnv, .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunctionMerge, - .finalizeFunc = elapsedFinalize + .finalizeFunc = elapsedFinalize, + .invertFunc = NULL, + .combineFunc = elapsedCombine, }, { .name = "last_row", @@ -1608,14 +1628,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, .processFunc = histogramFunction, .finalizeFunc = histogramFinalize, + .invertFunc = NULL, + .combineFunc = histogramCombine, .pPartialFunc = "_histogram_partial", - .pMergeFunc = "_histogram_merge" + .pMergeFunc = "_histogram_merge", }, { .name = "_histogram_partial", @@ -1625,7 +1647,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, .processFunc = histogramFunction, - .finalizeFunc = histogramPartialFinalize + .finalizeFunc = histogramPartialFinalize, + .invertFunc = NULL, + .combineFunc = histogramCombine, }, { .name = "_histogram_merge", @@ -1635,7 +1659,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHistogramFuncEnv, .initFunc = functionSetup, .processFunc = histogramFunctionMerge, - .finalizeFunc = histogramFinalize + .finalizeFunc = histogramFinalize, + .invertFunc = NULL, + .combineFunc = histogramCombine, }, { .name = "hyperloglog", @@ -1646,6 +1672,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = hllFunction, .finalizeFunc = hllFinalize, + .invertFunc = NULL, + .combineFunc = hllCombine, .pPartialFunc = "_hyperloglog_partial", .pMergeFunc = "_hyperloglog_merge" }, @@ -1657,7 +1685,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, .processFunc = hllFunction, - .finalizeFunc = hllPartialFinalize + .finalizeFunc = hllPartialFinalize, + .invertFunc = NULL, + .combineFunc = hllCombine, }, { .name = "_hyperloglog_merge", @@ -1667,7 +1697,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, .processFunc = hllFunctionMerge, - .finalizeFunc = hllFinalize + .finalizeFunc = hllFinalize, + .invertFunc = NULL, + .combineFunc = hllCombine, }, { .name = "diff", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1eafd3c649..812985f92a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -142,10 +142,8 @@ typedef struct SElapsedInfo { typedef struct SHistoFuncBin { double lower; double upper; - union { - int64_t count; - double percentage; - }; + int64_t count; + double percentage; } SHistoFuncBin; typedef struct SHistoFuncInfo { @@ -2103,8 +2101,49 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* pOutput) { + pOutput->percent = pInput->percent; + pOutput->algo = pInput->algo; + if (pOutput->algo == APERCT_ALGO_TDIGEST) { + buildTDigestInfo(pInput); + tdigestAutoFill(pInput->pTDigest, COMPRESSION); + + if(pInput->pTDigest->num_centroids == 0 && pInput->pTDigest->num_buffered_pts == 0) { + return; + } + + buildTDigestInfo(pOutput); + TDigest *pTDigest = pOutput->pTDigest; + + if(pTDigest->num_centroids <= 0) { + memcpy(pTDigest, pInput->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); + tdigestAutoFill(pTDigest, COMPRESSION); + } else { + tdigestMerge(pTDigest, pInput->pTDigest); + } + } else { + buildHistogramInfo(pInput); + if (pInput->pHisto->numOfElems <= 0) { + return; + } + + buildHistogramInfo(pOutput); + SHistogramInfo *pHisto = pOutput->pHisto; + + if (pHisto->numOfElems <= 0) { + memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); + } else { + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); + memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); + tHistogramDestroy(&pRes); + } + } +} + int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { - int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -2113,60 +2152,14 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - SAPercentileInfo* pInputInfo; int32_t start = pInput->startRowIndex; - for (int32_t i = start; i < pInput->numOfRows + start; ++i) { - //if (colDataIsNull_s(pCol, i)) { - // continue; - //} - numOfElems += 1; - char* data = colDataGetData(pCol, i); + char* data = colDataGetData(pCol, start); + SAPercentileInfo* pInputInfo = (SAPercentileInfo *)varDataVal(data); - pInputInfo = (SAPercentileInfo *)varDataVal(data); - } + apercentileTransferInfo(pInputInfo, pInfo); - pInfo->percent = pInputInfo->percent; - pInfo->algo = pInputInfo->algo; - if (pInfo->algo == APERCT_ALGO_TDIGEST) { - buildTDigestInfo(pInputInfo); - tdigestAutoFill(pInputInfo->pTDigest, COMPRESSION); - - if(pInputInfo->pTDigest->num_centroids == 0 && pInputInfo->pTDigest->num_buffered_pts == 0) { - return TSDB_CODE_SUCCESS; - } - - buildTDigestInfo(pInfo); - TDigest *pTDigest = pInfo->pTDigest; - - if(pTDigest->num_centroids <= 0) { - memcpy(pTDigest, pInputInfo->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); - tdigestAutoFill(pTDigest, COMPRESSION); - } else { - tdigestMerge(pTDigest, pInputInfo->pTDigest); - } - } else { - buildHistogramInfo(pInputInfo); - if (pInputInfo->pHisto->numOfElems <= 0) { - return TSDB_CODE_SUCCESS; - } - - buildHistogramInfo(pInfo); - SHistogramInfo *pHisto = pInfo->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInputInfo->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInputInfo->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - } - - SET_VAL(pResInfo, numOfElems, 1); + SET_VAL(pResInfo, 1, 1); return TSDB_CODE_SUCCESS; } @@ -2232,7 +2225,6 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SAPercentileInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SAPercentileInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); @@ -2850,6 +2842,7 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { + offset += pCol->info.bytes; continue; } @@ -3049,6 +3042,17 @@ _spread_over: return TSDB_CODE_SUCCESS; } +static void spreadTransferInfo(SSpreadInfo* pInput, SSpreadInfo* pOutput) { + pOutput->hasResult = pInput->hasResult; + if (pInput->max > pOutput->max) { + pOutput->max = pInput->max; + } + + if (pInput->min < pOutput->min) { + pOutput->min = pInput->min; + } +} + int32_t spreadFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3061,14 +3065,7 @@ int32_t spreadFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); pInputInfo = (SSpreadInfo *)varDataVal(data); - pInfo->hasResult = pInputInfo->hasResult; - if (pInputInfo->max > pInfo->max) { - pInfo->max = pInputInfo->max; - } - - if (pInputInfo->min < pInfo->min) { - pInfo->min = pInputInfo->min; - } + spreadTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), 1, 1); @@ -3101,6 +3098,17 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); + SSpreadInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + SSpreadInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + spreadTransferInfo(pSBuf, pDBuf); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + return TSDB_CODE_SUCCESS; +} + int32_t getElapsedInfoSize() { return (int32_t)sizeof(SElapsedInfo); } @@ -3206,6 +3214,17 @@ _elapsed_over: return TSDB_CODE_SUCCESS; } +static void elapsedTransferInfo(SElapsedInfo* pInput, SElapsedInfo* pOutput) { + pOutput->timeUnit = pInput->timeUnit; + if (pOutput->min > pInput->min) { + pOutput->min = pInput->min; + } + + if (pOutput->max < pInput->max) { + pOutput->max = pInput->max; + } +} + int32_t elapsedFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3217,14 +3236,7 @@ int32_t elapsedFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); SElapsedInfo* pInputInfo = (SElapsedInfo *)varDataVal(data); - pInfo->timeUnit = pInputInfo->timeUnit; - if (pInfo->min > pInputInfo->min) { - pInfo->min = pInputInfo->min; - } - - if (pInfo->max < pInputInfo->max) { - pInfo->max = pInputInfo->max; - } + elapsedTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; @@ -3256,6 +3268,18 @@ int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); + SElapsedInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + SElapsedInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + + elapsedTransferInfo(pSBuf, pDBuf); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + return TSDB_CODE_SUCCESS; +} + int32_t getHistogramInfoSize() { return (int32_t)sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin); } @@ -3470,6 +3494,17 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } +static void histogramTransferInfo(SHistoFuncInfo* pInput, SHistoFuncInfo* pOutput) { + pOutput->normalized = pInput->normalized; + pOutput->numOfBins = pInput->numOfBins; + pOutput->totalCount += pInput->totalCount; + for (int32_t k = 0; k < pOutput->numOfBins; ++k) { + pOutput->bins[k].lower = pInput->bins[k].lower; + pOutput->bins[k].upper = pInput->bins[k].upper; + pOutput->bins[k].count += pInput->bins[k].count; + } +} + int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3481,14 +3516,7 @@ int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); SHistoFuncInfo* pInputInfo = (SHistoFuncInfo *)varDataVal(data); - pInfo->normalized = pInputInfo->normalized; - pInfo->numOfBins = pInputInfo->numOfBins; - pInfo->totalCount += pInputInfo->totalCount; - for (int32_t k = 0; k < pInfo->numOfBins; ++k) { - pInfo->bins[k].lower = pInputInfo->bins[k].lower; - pInfo->bins[k].upper = pInputInfo->bins[k].upper; - pInfo->bins[k].count += pInputInfo->bins[k].count; - } + histogramTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), pInfo->numOfBins, pInfo->numOfBins); return TSDB_CODE_SUCCESS; @@ -3547,6 +3575,18 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return 1; } +int32_t histogramCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); + SHistoFuncInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + SHistoFuncInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + + histogramTransferInfo(pSBuf, pDBuf); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + return TSDB_CODE_SUCCESS; +} + int32_t getHLLInfoSize() { return (int32_t)sizeof(SHLLInfo); } @@ -3676,6 +3716,14 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } +static void hllTransferInfo(SHLLInfo* pInput, SHLLInfo* pOutput) { + for (int32_t k = 0; k < HLL_BUCKETS; ++k) { + if (pOutput->buckets[k] < pInput->buckets[k]) { + pOutput->buckets[k] = pInput->buckets[k]; + } + } +} + int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3687,11 +3735,7 @@ int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); SHLLInfo* pInputInfo = (SHLLInfo *)varDataVal(data); - for (int32_t k = 0; k < HLL_BUCKETS; ++k) { - if (pInfo->buckets[k] < pInputInfo->buckets[k]) { - pInfo->buckets[k] = pInputInfo->buckets[k]; - } - } + hllTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; @@ -3727,6 +3771,18 @@ int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); + SHLLInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + SHLLInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + + hllTransferInfo(pSBuf, pDBuf); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + return TSDB_CODE_SUCCESS; +} + bool getStateFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SStateInfo); return true; @@ -4130,6 +4186,8 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { SColumnInfoData* pInputCol = pInput->pData[0]; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + int32_t alreadySampled = pInfo->numSampled; + int32_t startOffset = pCtx->offset; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_s(pInputCol, i)) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index f2514f54f1..df09d3e529 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -159,6 +159,8 @@ bool fmIsRepeatScanFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, bool fmIsUserDefinedFunc(int32_t funcId) { return funcId > FUNC_UDF_ID_START; } +bool fmIsForbidFillFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_FILL_FUNC); } + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c60cebb95c..f7e65f100a 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2816,10 +2816,85 @@ static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableIndexInfoIntervalUnit = "IntervalUnit"; +static const char* jkTableIndexInfoSlidingUnit = "SlidingUnit"; +static const char* jkTableIndexInfoInterval = "Interval"; +static const char* jkTableIndexInfoOffset = "Offset"; +static const char* jkTableIndexInfoSliding = "Sliding"; +static const char* jkTableIndexInfoDstTbUid = "DstTbUid"; +static const char* jkTableIndexInfoDstVgId = "DstVgId"; +static const char* jkTableIndexInfoEpSet = "EpSet"; +static const char* jkTableIndexInfoExpr = "Expr"; + +static int32_t tableIndexInfoToJson(const void* pObj, SJson* pJson) { + const STableIndexInfo* pNode = (const STableIndexInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoIntervalUnit, pNode->intervalUnit); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoSlidingUnit, pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoInterval, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoOffset, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoSliding, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoDstTbUid, pNode->dstTbUid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoDstVgId, pNode->dstVgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTableIndexInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableIndexInfoExpr, pNode->expr); + } + + return code; +} + +static int32_t jsonToTableIndexInfo(const SJson* pJson, void* pObj) { + STableIndexInfo* pNode = (STableIndexInfo*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkTableIndexInfoIntervalUnit, &pNode->intervalUnit); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkTableIndexInfoSlidingUnit, &pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoInterval, &pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoOffset, &pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoSliding, &pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoDstTbUid, &pNode->dstTbUid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTableIndexInfoDstVgId, &pNode->dstVgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkTableIndexInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonDupStringValue(pJson, jkTableIndexInfoExpr, &pNode->expr); + } + + return code; +} + static const char* jkRealTableMetaSize = "MetaSize"; static const char* jkRealTableMeta = "Meta"; static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; static const char* jkRealTableVgroupsInfo = "VgroupsInfo"; +static const char* jkRealTableSmaIndexes = "SmaIndexes"; static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { const SRealTableNode* pNode = (const SRealTableNode*)pObj; @@ -2837,6 +2912,9 @@ static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddTArray(pJson, jkRealTableSmaIndexes, tableIndexInfoToJson, pNode->pSmaIndexes); + } return code; } @@ -2858,6 +2936,10 @@ static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); } + if (TSDB_CODE_SUCCESS == code) { + code = + tjsonToTArray(pJson, jkRealTableSmaIndexes, jsonToTableIndexInfo, &pNode->pSmaIndexes, sizeof(STableIndexInfo)); + } return code; } @@ -4214,6 +4296,7 @@ int32_t nodesStringToList(const char* pStr, SNodeList** pList) { tjsonDelete(pJson); if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(*pList); + *pList = NULL; terrno = code; return code; } diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 3efe6700d2..adedcf0fd9 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -24,6 +24,9 @@ extern "C" { #include "parUtil.h" #include "parser.h" +#define QUERY_SMA_OPTIMIZE_DISABLE 0 +#define QUERY_SMA_OPTIMIZE_ENABLE 1 + int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery); int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 0351023f5b..fb67a35368 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -46,6 +46,7 @@ typedef struct SParseMetaCache { SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass SHashObj* pUdf; // key is funcName, element is SFuncInfo* + SHashObj* pTableIndex; // key is tbFName, element is SArray* } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -58,7 +59,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf); +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMsgBuf* pMsgBuf); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); @@ -75,6 +76,7 @@ int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pD SParseMetaCache* pMetaCache); int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); +int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta); int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); @@ -84,6 +86,7 @@ int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDb int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, bool* pPass); int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo); +int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 613a2d867d..e7f4b05bc8 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -346,25 +346,30 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera return (SNode*)val; } +static int32_t addParamToLogicConditionNode(SLogicConditionNode* pCond, SNode* pParam) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam) && pCond->condType == ((SLogicConditionNode*)pParam)->condType) { + int32_t code = nodesListAppendList(pCond->pParameterList, ((SLogicConditionNode*)pParam)->pParameterList); + ((SLogicConditionNode*)pParam)->pParameterList = NULL; + nodesDestroyNode(pParam); + return code; + } else { + return nodesListAppend(pCond->pParameterList, pParam); + } +} + SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2) { CHECK_PARSER_STATUS(pCxt); SLogicConditionNode* cond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); CHECK_OUT_OF_MEM(cond); cond->condType = type; cond->pParameterList = nodesMakeList(); - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1) && type == ((SLogicConditionNode*)pParam1)->condType) { - nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam1)->pParameterList); - ((SLogicConditionNode*)pParam1)->pParameterList = NULL; - nodesDestroyNode(pParam1); - } else { - nodesListAppend(cond->pParameterList, pParam1); + int32_t code = addParamToLogicConditionNode(cond, pParam1); + if (TSDB_CODE_SUCCESS == code && NULL != pParam2) { + code = addParamToLogicConditionNode(cond, pParam2); } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2) && type == ((SLogicConditionNode*)pParam2)->condType) { - nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam2)->pParameterList); - ((SLogicConditionNode*)pParam2)->pParameterList = NULL; - nodesDestroyNode(pParam2); - } else { - nodesListAppend(cond->pParameterList, pParam2); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(cond); + return NULL; } return (SNode*)cond; } @@ -796,7 +801,8 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti ((SDatabaseOptions*)pOptions)->pRetentions = pVal; break; case DB_OPTION_SCHEMALESS: - ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); +// ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->schemaless = 1; break; default: break; diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 26d4b69fa0..aca7d9c9d3 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -19,6 +19,7 @@ #include "parInt.h" #include "parToken.h" #include "systable.h" +#include "tglobal.h" typedef void* (*FMalloc)(size_t); typedef void (*FFree)(void*); @@ -91,6 +92,7 @@ abort_parse: typedef struct SCollectMetaKeyCxt { SParseContext* pParseCxt; SParseMetaCache* pMetaCache; + SNode* pStmt; } SCollectMetaKeyCxt; static void destroyCollectMetaKeyCxt(SCollectMetaKeyCxt* pCxt) { @@ -114,6 +116,14 @@ static EDealRes collectMetaKeyFromFunction(SCollectMetaKeyFromExprCxt* pCxt, SFu return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } +static bool needGetTableIndex(SNode* pStmt) { + if (QUERY_SMA_OPTIMIZE_ENABLE == tsQuerySmaOptimize && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt; + return (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)); + } + return false; +} + static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTableNode* pRealTable, AUTH_TYPE authType) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, @@ -129,6 +139,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTa if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code && needGetTableIndex(pCxt->pStmt)) { + code = reserveTableIndexInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, + pCxt->pMetaCache); + } return code; } @@ -379,6 +393,7 @@ static int32_t collectMetaKeyFromDelete(SCollectMetaKeyCxt* pCxt, SDeleteStmt* p } static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { + pCxt->pStmt = pStmt; switch (nodeType(pStmt)) { case QUERY_NODE_SET_OPERATOR: return collectMetaKeyFromSetOperator(pCxt, (SSetOperator*)pStmt); @@ -449,7 +464,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { } int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery) { - SCollectMetaKeyCxt cxt = {.pParseCxt = pParseCxt, .pMetaCache = taosMemoryCalloc(1, sizeof(SParseMetaCache))}; + SCollectMetaKeyCxt cxt = { + .pParseCxt = pParseCxt, .pMetaCache = taosMemoryCalloc(1, sizeof(SParseMetaCache)), .pStmt = pQuery->pRoot}; if (NULL == cxt.pMetaCache) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 54c8a18218..c721491489 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1297,11 +1297,12 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { } static int32_t checkSchemalessDb(SInsertParseContext* pCxt, char* pDbName) { - SDbCfgInfo pInfo = {0}; - char fullName[TSDB_TABLE_FNAME_LEN]; - snprintf(fullName, sizeof(fullName), "%d.%s", pCxt->pComCxt->acctId, pDbName); - CHECK_CODE(getDBCfg(pCxt, fullName, &pInfo)); - return pInfo.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; +// SDbCfgInfo pInfo = {0}; +// char fullName[TSDB_TABLE_FNAME_LEN]; +// snprintf(fullName, sizeof(fullName), "%d.%s", pCxt->pComCxt->acctId, pDbName); +// CHECK_CODE(getDBCfg(pCxt, fullName, &pInfo)); +// return pInfo.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } // tb_name @@ -2119,9 +2120,11 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS isOrdered = false; } if (index < 0) { + uError("smlBoundColumnData. index:%d", index); return TSDB_CODE_SML_INVALID_DATA; } if (pColList->cols[index].valStat == VAL_STAT_HAS) { + uError("smlBoundColumnData. already set. index:%d", index); return TSDB_CODE_SML_INVALID_DATA; } lastColIdx = index; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 538db81588..84bded8a58 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -41,6 +41,7 @@ typedef struct STranslateContext { SHashObj* pTables; SExplainOptions* pExplainOpt; SParseMetaCache* pMetaCache; + bool createStream; } STranslateContext; typedef struct SFullDatabaseName { @@ -255,6 +256,23 @@ static int32_t getUdfInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { return code; } +static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray** pIndexes) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } + if (pParCxt->async) { + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); + } else { + code = catalogGetTableIndex(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pIndexes); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname); + } + return code; +} + static int32_t initTranslateContext(SParseContext* pParseCxt, SParseMetaCache* pMetaCache, STranslateContext* pCxt) { pCxt->pParseCxt = pParseCxt; pCxt->errCode = TSDB_CODE_SUCCESS; @@ -329,6 +347,10 @@ static bool isIndefiniteRowsFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsIndefiniteRowsFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isVectorFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsVectorFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrSelectStmt->isDistinct); } @@ -711,7 +733,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD } int32_t len = 0; - if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), + if (!taosMbsToUcs4(pVal->literal, strlen(pVal->literal), (TdUcs4*)varDataVal(pVal->datum.p), targetDt.bytes - VARSTR_HEADER_SIZE, &len)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } @@ -952,6 +974,9 @@ static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { } static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsAggFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (beforeHaving(pCxt->currClause)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); } @@ -969,6 +994,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { } static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (0 == LIST_LENGTH(pFunc->pParameterList)) { if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrSelectStmt->pFromTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); @@ -985,6 +1013,9 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN } static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsIndefiniteRowsFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc || pCxt->pCurrSelectStmt->hasAggFuncs) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); @@ -995,6 +1026,18 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod return TSDB_CODE_SUCCESS; } +static int32_t translateForbidFillFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsForbidFillFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (NULL != pCxt->pCurrSelectStmt->pWindow && + QUERY_NODE_INTERVAL_WINDOW == nodeType(pCxt->pCurrSelectStmt->pWindow) && + NULL != ((SIntervalWindowNode*)pCxt->pCurrSelectStmt->pWindow)->pFill) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) { if (NULL != pSelect) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); @@ -1012,15 +1055,18 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) } pCxt->errCode = getFuncInfo(pCxt, pFunc); - if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsAggFunc(pFunc->funcId)) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = translateAggFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsScanPseudoColumnFunc(pFunc->funcId)) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = translateScanPseudoColumnFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsIndefiniteRowsFunc(pFunc->funcId)) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = translateIndefiniteRowsFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = translateForbidFillFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == pCxt->errCode) { setFuncClassification(pCxt->pCurrSelectStmt, pFunc); } @@ -1364,6 +1410,17 @@ static bool isSingleTable(SRealTableNode* pRealTable) { return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } +static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + if (pCxt->createStream || QUERY_SMA_OPTIMIZE_DISABLE == tsQuerySmaOptimize) { + return TSDB_CODE_SUCCESS; + } + if (NULL != pCxt->pCurrSelectStmt && NULL != pCxt->pCurrSelectStmt->pWindow && + QUERY_NODE_INTERVAL_WINDOW == nodeType(pCxt->pCurrSelectStmt->pWindow)) { + return getTableIndex(pCxt, pName, &pRealTable->pSmaIndexes); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -1380,6 +1437,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); } code = setTableVgroupList(pCxt, &name, pRealTable); + if (TSDB_CODE_SUCCESS == code) { + code = setTableIndex(pCxt, &name, pRealTable); + } } pRealTable->table.precision = pRealTable->pMeta->tableInfo.precision; pRealTable->table.singleTable = isSingleTable(pRealTable); @@ -1803,7 +1863,7 @@ static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char uni return -1; } -static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { +static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; @@ -1845,7 +1905,15 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SInte } } - return translateFill(pCxt, pWhere, pInterval); + return TSDB_CODE_SUCCESS; +} + +static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSelect, SIntervalWindowNode* pInterval) { + int32_t code = checkIntervalWindow(pCxt, pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = translateFill(pCxt, pSelect->pWhere, pInterval); + } + return code; } static EDealRes checkStateExpr(SNode* pNode, void* pContext) { @@ -1867,13 +1935,13 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t checkStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { +static int32_t translateStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); // todo check for "function not support for state_window" return pCxt->errCode; } -static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* pSession) { +static int32_t translateSessionWindow(STranslateContext* pCxt, SSessionWindowNode* pSession) { if ('y' == pSession->pGap->unit || 'n' == pSession->pGap->unit || 0 == pSession->pGap->datum.i) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SESSION_GAP); } @@ -1884,14 +1952,14 @@ static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* p return TSDB_CODE_SUCCESS; } -static int32_t checkWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return checkStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); + return translateStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); case QUERY_NODE_SESSION_WINDOW: - return checkSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); + return translateSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: - return checkIntervalWindow(pCxt, pSelect->pWhere, (SIntervalWindowNode*)pSelect->pWindow); + return translateIntervalWindow(pCxt, pSelect, (SIntervalWindowNode*)pSelect->pWindow); default: break; } @@ -1905,7 +1973,7 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { - code = checkWindow(pCxt, pSelect); + code = translateSpecificWindow(pCxt, pSelect); } return code; } @@ -2321,7 +2389,9 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); } - SNode* pRetention = NULL; + SValueNode* pPrevFreq = NULL; + SValueNode* pPrevKeep = NULL; + SNode* pRetention = NULL; FOREACH(pRetention, pRetentions) { SNode* pNode = NULL; FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { @@ -2330,6 +2400,16 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return pCxt->errCode; } } + + SValueNode* pFreq = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 0); + SValueNode* pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 1); + if (pFreq->datum.i <= 0 || 'n' == pFreq->unit || 'y' == pFreq->unit || pFreq->datum.i >= pKeep->datum.i || + (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) || + (NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + } + pPrevFreq = pFreq; + pPrevKeep = pKeep; } return TSDB_CODE_SUCCESS; @@ -2549,12 +2629,23 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS return TSDB_CODE_SUCCESS; } +static bool validRollupFunc(const char* pFunc) { + static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"}; + static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*)); + for (int i = 0; i < numOfRollupFuncs; ++i) { + if (0 == strcmp(rollupFuncs[i], pFunc)) { + return true; + } + } + return false; +} + static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { if (NULL == pFuncs) { return TSDB_CODE_SUCCESS; } - if (1 != LIST_LENGTH(pFuncs)) { + if (1 != LIST_LENGTH(pFuncs) || !validRollupFunc(((SFunctionNode*)nodesListGetNode(pFuncs, 0))->functionName)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); } return TSDB_CODE_SUCCESS; @@ -2671,15 +2762,16 @@ static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt } static int32_t checkSchemalessDb(STranslateContext* pCxt, const char* pDbName) { - if (0 != pCxt->pParseCxt->schemalessType) { - return TSDB_CODE_SUCCESS; - } - SDbCfgInfo info = {0}; - int32_t code = getDBCfg(pCxt, pDbName, &info); - if (TSDB_CODE_SUCCESS == code) { - code = info.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; - } - return code; +// if (0 != pCxt->pParseCxt->schemalessType) { +// return TSDB_CODE_SUCCESS; +// } +// SDbCfgInfo info = {0}; +// int32_t code = getDBCfg(pCxt, pDbName, &info); +// if (TSDB_CODE_SUCCESS == code) { +// code = info.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; +// } +// return code; + return TSDB_CODE_SUCCESS; } static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { @@ -2724,7 +2816,8 @@ typedef struct SSampleAstInfo { STableMeta* pRollupTableMeta; } SSampleAstInfo; -static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen) { +static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, + int32_t* pExprLen) { SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT); if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2769,10 +2862,14 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; strcpy(((SColumnNode*)pInterval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); + pCxt->createStream = true; int32_t code = translateQuery(pCxt, (SNode*)pSelect); if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString(pSelect, false, pAst, pLen); } + if (TSDB_CODE_SUCCESS == code && NULL != pExpr) { + code = nodesListToString(pSelect->pProjectionList, false, pExpr, pExprLen); + } nodesDestroyNode(pSelect); return code; } @@ -2894,7 +2991,7 @@ static int32_t getRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SR SSampleAstInfo info = {0}; int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info); if (TSDB_CODE_SUCCESS == code) { - code = buildSampleAst(pCxt, &info, pAst, pLen); + code = buildSampleAst(pCxt, &info, pAst, pLen, NULL, NULL); } clearSampleAstInfo(&info); return code; @@ -3034,15 +3131,14 @@ static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pSt SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), alterReq.name); alterReq.alterType = pStmt->alterType; - if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == pStmt->alterType) { - return TSDB_CODE_FAILED; - } else { - if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) { - return TSDB_CODE_OUT_OF_MEMORY; - } + if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == pStmt->alterType || TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME == pStmt->alterType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); } - - return buildCmdMsg(pCxt, TDMT_MND_ALTER_STB, (FSerializeFunc)tSerializeSMAlterStbReq, &alterReq); + int32_t code = setAlterTableField(pStmt, &alterReq); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_ALTER_STB, (FSerializeFunc)tSerializeSMAlterStbReq, &alterReq); + } + return code; } static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { @@ -3122,7 +3218,7 @@ static int32_t nodeTypeToShowType(ENodeType nt) { case QUERY_NODE_SHOW_QUERIES_STMT: return TSDB_MGMT_TABLE_QUERIES; case QUERY_NODE_SHOW_VARIABLE_STMT: - return 0; // todo + return TSDB_MGMT_TABLE_CONFIGS; default: break; } @@ -3152,10 +3248,6 @@ static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLe return TSDB_CODE_SUCCESS; } -static int32_t getSmaIndexExpr(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pExpr, int32_t* pLen) { - return nodesListToString(pStmt->pOptions->pFuncs, false, pExpr, pLen); -} - static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SSampleAstInfo* pInfo) { pInfo->pDbName = pCxt->pParseCxt->db; pInfo->pTableName = pStmt->tableName; @@ -3171,11 +3263,12 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt return TSDB_CODE_SUCCESS; } -static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { +static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen, + char** pExpr, int32_t* pExprLen) { SSampleAstInfo info = {0}; int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info); if (TSDB_CODE_SUCCESS == code) { - code = buildSampleAst(pCxt, &info, pAst, pLen); + code = buildSampleAst(pCxt, &info, pAst, pLen, pExpr, pExprLen); } clearSampleAstInfo(&info); return code; @@ -3201,10 +3294,7 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen); } if (TSDB_CODE_SUCCESS == code) { - code = getSmaIndexExpr(pCxt, pStmt, &pReq->expr, &pReq->exprLen); - } - if (TSDB_CODE_SUCCESS == code) { - code = getSmaIndexAst(pCxt, pStmt, &pReq->ast, &pReq->astLen); + code = getSmaIndexAst(pCxt, pStmt, &pReq->ast, &pReq->astLen, &pReq->expr, &pReq->exprLen); } return code; @@ -3487,6 +3577,7 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* tNameExtractFullName(&name, pReq->targetStbFullName); } + pCxt->createStream = true; int32_t code = translateQuery(pCxt, pStmt->pQuery); if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); @@ -3734,6 +3825,7 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: code = translateShow(pCxt, (SShowStmt*)pNode); break; case QUERY_NODE_CREATE_INDEX_STMT: @@ -4888,7 +4980,11 @@ static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, case TSDB_ALTER_TABLE_UPDATE_OPTIONS: return buildUpdateOptionsReq(pCxt, pStmt, pReq); case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - return buildRenameColReq(pCxt, pStmt, pTableMeta, pReq); + if (TSDB_CHILD_TABLE == pTableMeta->tableType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + } else { + return buildRenameColReq(pCxt, pStmt, pTableMeta, pReq); + } default: break; } @@ -4953,6 +5049,9 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; int32_t code = checkSchemalessDb(pCxt, pStmt->dbName); + if (TSDB_CODE_SUCCESS != code) { + return code; + } STableMeta* pTableMeta = NULL; code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 29443876a6..716b120af5 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -76,7 +76,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_KEEP_ORDER: return "Invalid keep value, should be keep0 <= keep1 <= keep2"; case TSDB_CODE_PAR_INVALID_KEEP_VALUE: - return "Invalid option keep: %d, %d, %d valid range: [%d, %d]"; + return "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]"; case TSDB_CODE_PAR_INVALID_COMMENT_OPTION: return "Invalid option comment, length cannot exceed %d"; case TSDB_CODE_PAR_INVALID_F_RANGE_OPTION: @@ -182,6 +182,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "The DELETE statement must have a definite time window range"; case TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG: return "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes"; + case TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC: + return "%s function not allowed in fill query"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -542,6 +544,9 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog if (TSDB_CODE_SUCCESS == code) { code = buildUdfReq(pMetaCache->pUdf, &pCatalogReq->pUdf); } + if (TSDB_CODE_SUCCESS == code) { + code = buildTableReq(pMetaCache->pTableIndex, &pCatalogReq->pTableIndex); + } return code; } @@ -628,6 +633,9 @@ int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMet if (TSDB_CODE_SUCCESS == code) { code = putUdfToCache(pCatalogReq->pUdf, pMetaData->pUdfList, pMetaCache->pUdf); } + if (TSDB_CODE_SUCCESS == code) { + code = putTableDataToCache(pCatalogReq->pTableIndex, pMetaData->pTableIndex, pMetaCache->pTableIndex); + } return code; } @@ -806,3 +814,43 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun } return code; } + +static void destroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } + +static SArray* smaIndexesDup(SArray* pSrc) { + SArray* pDst = taosArrayDup(pSrc); + if (NULL == pDst) { + return NULL; + } + int32_t size = taosArrayGetSize(pDst); + for (int32_t i = 0; i < size; ++i) { + ((STableIndexInfo*)taosArrayGet(pDst, i))->expr = NULL; + } + for (int32_t i = 0; i < size; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pDst, i); + pIndex->expr = taosMemoryStrDup(((STableIndexInfo*)taosArrayGet(pSrc, i))->expr); + if (NULL == pIndex->expr) { + taosArrayDestroyEx(pDst, destroySmaIndex); + return NULL; + } + } + return pDst; +} + +int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { + return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableIndex); +} + +int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pName, fullName); + SArray* pSmaIndexes = NULL; + int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableIndex, (void**)&pSmaIndexes); + if (TSDB_CODE_SUCCESS == code && NULL != pSmaIndexes) { + *pIndexes = smaIndexesDup(pSmaIndexes); + if (NULL == *pIndexes) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + return code; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index c2e1eba472..edb0d79f02 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -76,7 +76,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.bytes = inputSize; - + switch (pParam->buffer_type) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -186,7 +186,7 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p return extractResultSchema(pRoot, numOfCols, pSchema); } -int32_t qSetSTableIdForRSma(SNode* pStmt, int64_t uid) { +int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { SNode* pTable = ((SSelectStmt*)pStmt)->pFromTable; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index c7f0990132..0b2aacf5ef 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -214,6 +214,11 @@ int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, void* pTransporter, con int32_t __catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { return 0; } +int32_t __catalogGetTableIndex(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pName, + SArray** pRes) { + return g_mockCatalogService->catalogGetTableIndex(pName, pRes); +} + void initMetaDataEnv() { g_mockCatalogService.reset(new MockCatalogService()); @@ -230,6 +235,7 @@ void initMetaDataEnv() { stub.set(catalogGetUdfInfo, __catalogGetUdfInfo); stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta); stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta); + stub.set(catalogGetTableIndex, __catalogGetTableIndex); // { // AddrAny any("libcatalog.so"); // std::map result; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 0c37c875c0..010d4a2f72 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -149,6 +149,22 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } + int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const { + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(pTableName, tbFName); + auto it = index_.find(tbFName); + if (index_.end() == it) { + return TSDB_CODE_SUCCESS; + } + *pIndexes = taosArrayInit(it->second.size(), sizeof(STableIndexInfo)); + for (const auto& index : it->second) { + STableIndexInfo info; + + taosArrayPush(*pIndexes, copyTableIndexInfo(&info, &index)); + } + return TSDB_CODE_SUCCESS; + } + int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { int32_t code = getAllTableMeta(pCatalogReq->pTableMeta, &pMetaData->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -169,6 +185,9 @@ class MockCatalogServiceImpl { if (TSDB_CODE_SUCCESS == code) { code = getAllUdf(pCatalogReq->pUdf, &pMetaData->pUdfList); } + if (TSDB_CODE_SUCCESS == code) { + code = getAllTableIndex(pCatalogReq->pTableIndex, &pMetaData->pTableIndex); + } return code; } @@ -176,7 +195,7 @@ class MockCatalogServiceImpl { int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); - meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->uid = getNextId(); return *(builder_.get()); } @@ -187,14 +206,11 @@ class MockCatalogServiceImpl { } meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema = table.release(); - meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->uid = getNextId(); meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; SVgroupInfo vgroup = {vgid, 0, 0, {0}, 0}; - addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); - addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); - addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); - vgroup.epSet.inUse = 0; + genEpSet(&vgroup.epSet); meta_[db][tbname]->vgs.emplace_back(vgroup); // super table @@ -268,10 +284,45 @@ class MockCatalogServiceImpl { udf_.insert(std::make_pair(func, info)); } + void createSmaIndex(const SMCreateSmaReq* pReq) { + STableIndexInfo info; + info.intervalUnit = pReq->intervalUnit; + info.slidingUnit = pReq->slidingUnit; + info.interval = pReq->interval; + info.offset = pReq->offset; + info.sliding = pReq->sliding; + info.dstTbUid = getNextId(); + info.dstVgId = pReq->dstVgId; + genEpSet(&info.epSet); + info.expr = strdup(pReq->expr); + auto it = index_.find(pReq->stb); + if (index_.end() == it) { + index_.insert(std::make_pair(std::string(pReq->stb), std::vector{info})); + } else { + it->second.push_back(info); + } + } + private: typedef std::map> TableMetaCache; typedef std::map DbMetaCache; typedef std::map> UdfMetaCache; + typedef std::map> IndexMetaCache; + + uint64_t getNextId() { return id_++; } + + void genEpSet(SEpSet* pEpSet) { + addEpIntoEpSet(pEpSet, "dnode_1", 6030); + addEpIntoEpSet(pEpSet, "dnode_2", 6030); + addEpIntoEpSet(pEpSet, "dnode_3", 6030); + pEpSet->inUse = 0; + } + + STableIndexInfo* copyTableIndexInfo(STableIndexInfo* pDst, const STableIndexInfo* pSrc) const { + memcpy(pDst, pSrc, sizeof(STableIndexInfo)); + pDst->expr = strdup(pSrc->expr); + return pDst; + } std::string toDbname(const std::string& dbFullName) const { std::string::size_type n = dbFullName.find("."); @@ -463,10 +514,24 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } + int32_t getAllTableIndex(SArray* pTableIndex, SArray** pTableIndexData) const { + if (NULL != pTableIndex) { + int32_t num = taosArrayGetSize(pTableIndex); + *pTableIndexData = taosArrayInit(num, sizeof(SMetaRes)); + for (int32_t i = 0; i < num; ++i) { + SMetaRes res = {0}; + res.code = catalogGetTableIndex((const SName*)taosArrayGet(pTableIndex, i), (SArray**)(&res.pRes)); + taosArrayPush(*pTableIndexData, &res); + } + } + return TSDB_CODE_SUCCESS; + } + uint64_t id_; std::unique_ptr builder_; DbMetaCache meta_; UdfMetaCache udf_; + IndexMetaCache index_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {} @@ -490,6 +555,8 @@ void MockCatalogService::createFunction(const std::string& func, int8_t funcType impl_->createFunction(func, funcType, outputType, outputLen, bufSize); } +void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); } + int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { return impl_->catalogGetTableMeta(pTableName, pTableMeta); } @@ -510,6 +577,10 @@ int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFunc return impl_->catalogGetUdfInfo(funcName, pInfo); } +int32_t MockCatalogService::catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const { + return impl_->catalogGetTableIndex(pTableName, pIndexes); +} + int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 133a355c59..c4ab091b7a 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -57,12 +57,14 @@ class MockCatalogService { void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize); + void createSmaIndex(const SMCreateSmaReq* pReq); int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const; int32_t catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const; + int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const; int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const; private: diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index 22b244145b..f554651b90 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -24,7 +24,7 @@ class ParserInitialATest : public ParserDdlTest {}; TEST_F(ParserInitialATest, alterAccount) { useDb("root", "test"); - run("ALTER ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); + run("ALTER ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT, PARSER_STAGE_PARSE); } TEST_F(ParserInitialATest, alterDnode) { @@ -157,8 +157,8 @@ TEST_F(ParserInitialATest, alterSTable) { 20 + VARSTR_HEADER_SIZE); run("ALTER TABLE st1 MODIFY COLUMN c1 VARCHAR(20)"); - setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1"); - run("ALTER TABLE st1 RENAME COLUMN c1 cc1"); + // setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1"); + // run("ALTER TABLE st1 RENAME COLUMN c1 cc1"); setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT); run("ALTER TABLE st1 ADD TAG tag11 BIGINT"); @@ -177,6 +177,12 @@ TEST_F(ParserInitialATest, alterSTable) { // ADD {FULLTEXT | SMA} INDEX index_name (col_name [, col_name] ...) [index_option] } +TEST_F(ParserInitialATest, alterSTableSemanticCheck) { + useDb("root", "test"); + + run("ALTER TABLE st1 RENAME COLUMN c1 cc1", TSDB_CODE_PAR_INVALID_ALTER_TABLE); +} + TEST_F(ParserInitialATest, alterTable) { useDb("root", "test"); @@ -299,6 +305,12 @@ TEST_F(ParserInitialATest, alterTable) { // ADD {FULLTEXT | SMA} INDEX index_name (col_name [, col_name] ...) [index_option] } +TEST_F(ParserInitialATest, alterTableSemanticCheck) { + useDb("root", "test"); + + run("ALTER TABLE st1s1 RENAME COLUMN c1 cc1", TSDB_CODE_PAR_INVALID_ALTER_TABLE); +} + TEST_F(ParserInitialATest, alterUser) { useDb("root", "test"); @@ -323,7 +335,7 @@ TEST_F(ParserInitialATest, balanceVgroup) { TEST_F(ParserInitialATest, bug001) { useDb("root", "test"); - run("ALTER DATABASE db WAL 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR); + run("ALTER DATABASE db WAL 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR, PARSER_STAGE_PARSE); } } // namespace ParserTest \ No newline at end of file diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index d996ca196a..f306947f76 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -27,7 +27,7 @@ class ParserInitialCTest : public ParserDdlTest {}; TEST_F(ParserInitialCTest, createAccount) { useDb("root", "test"); - run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); + run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT, PARSER_STAGE_PARSE); } TEST_F(ParserInitialCTest, createBnode) { @@ -186,7 +186,7 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbReplicaFunc(3); addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); - addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 5, TIME_UNIT_MINUTE, TIME_UNIT_YEAR); + addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); setDbStrictaFunc(1); setDbWalLevelFunc(2); setDbVgroupsFunc(100); @@ -205,7 +205,7 @@ TEST_F(ParserInitialCTest, createDatabase) { "PAGESIZE 8 " "PRECISION 'ns' " "REPLICA 3 " - "RETENTIONS 15s:7d,1m:21d,15m:5y " + "RETENTIONS 15s:7d,1m:21d,15m:500d " "STRICT 1 " "WAL 2 " "VGROUPS 100 " @@ -220,6 +220,17 @@ TEST_F(ParserInitialCTest, createDatabase) { "KEEP 1440m,300h,400d "); } +TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) { + useDb("root", "test"); + + run("create database db2 retentions 0s:1d", TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + run("create database db2 retentions 10s:0d", TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + run("create database db2 retentions 1w:1d", TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + run("create database db2 retentions 1w:1n", TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + run("create database db2 retentions 15s:7d,15m:21d,10m:500d", TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + run("create database db2 retentions 15s:7d,5m:21d,10m:10d", TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); +} + TEST_F(ParserInitialCTest, createDnode) { useDb("root", "test"); @@ -434,6 +445,13 @@ TEST_F(ParserInitialCTest, createStable) { "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) FILE_FACTOR 0.1"); } +TEST_F(ParserInitialCTest, createStableSemanticCheck) { + useDb("root", "test"); + + run("CREATE STABLE stb2 (ts TIMESTAMP, c1 INT) TAGS (tag1 INT) ROLLUP(CEIL) FILE_FACTOR 0.1", + TSDB_CODE_PAR_INVALID_ROLLUP_OPTION, PARSER_STAGE_TRANSLATE); +} + TEST_F(ParserInitialCTest, createStream) { useDb("root", "test"); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 154e28a02c..51d302fe12 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -65,6 +65,8 @@ TEST_F(ParserSelectTest, condition) { run("SELECT c1 FROM t1 WHERE ts in (true, false)"); + run("SELECT c1 FROM t1 WHERE NOT ts in (true, false)"); + run("SELECT * FROM t1 WHERE c1 > 10 and c1 is not null"); } @@ -212,9 +214,11 @@ TEST_F(ParserSelectTest, interval) { TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); - run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, - PARSER_STAGE_TRANSLATE); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP); + run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 " + "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)", + TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, subquery) { diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index 07f3d3cece..ad21252c2b 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -36,7 +36,7 @@ class ParserTestBase : public testing::Test { void login(const std::string& user); void useDb(const std::string& acctId, const std::string& db); - void run(const std::string& sql, int32_t expect = TSDB_CODE_SUCCESS, ParserStage checkStage = PARSER_STAGE_ALL); + void run(const std::string& sql, int32_t expect = TSDB_CODE_SUCCESS, ParserStage checkStage = PARSER_STAGE_TRANSLATE); virtual void checkDdl(const SQuery* pQuery, ParserStage stage); diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 1a8c7657df..e0728c8654 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -36,11 +36,13 @@ extern "C" { #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); -int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList); +int32_t createColumnByRewriteExprs(SNodeList* pExprs, SNodeList** pList); +int32_t createColumnByRewriteExpr(SNode* pExpr, SNodeList** pList); +int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew); -int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); -int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); -int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); +int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan); +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a9f3909af6..d67b194192 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -220,6 +220,7 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT } TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); + TSWAP(pScan->pSmaIndexes, pRealTable->pSmaIndexes); pScan->tableId = pRealTable->pMeta->uid; pScan->stableId = pRealTable->pMeta->suid; pScan->tableType = pRealTable->pMeta->tableType; @@ -272,10 +273,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // set output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pScan->pScanCols, &pScan->node.pTargets); + code = createColumnByRewriteExprs(pScan->pScanCols, &pScan->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pScan->pScanPseudoCols, &pScan->node.pTargets); + code = createColumnByRewriteExprs(pScan->pScanPseudoCols, &pScan->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -441,10 +442,10 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, // set the output if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) { - code = createColumnByRewriteExps(pAgg->pGroupKeys, &pAgg->node.pTargets); + code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { - code = createColumnByRewriteExps(pAgg->pAggFuncs, &pAgg->node.pTargets); + code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -475,7 +476,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt // set the output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pIdfRowsFunc->pVectorFuncs, &pIdfRowsFunc->node.pTargets); + code = createColumnByRewriteExprs(pIdfRowsFunc->pVectorFuncs, &pIdfRowsFunc->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -505,7 +506,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pWindow->pFuncs, &pWindow->node.pTargets); + code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } pSelect->hasAggFuncs = false; @@ -776,7 +777,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe // set the output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pAgg->pGroupKeys, &pAgg->node.pTargets); + code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -918,7 +919,7 @@ static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pS // set the output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pAgg->pGroupKeys, &pAgg->node.pTargets); + code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -1040,7 +1041,7 @@ static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p // set output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pScan->pScanCols, &pScan->node.pTargets); + code = createColumnByRewriteExprs(pScan->pScanCols, &pScan->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -1064,7 +1065,7 @@ static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pD } // set the output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pAgg->pAggFuncs, &pAgg->node.pTargets); + code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -1138,11 +1139,40 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi return TSDB_CODE_FAILED; } -int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { - SLogicPlanContext cxt = {.pPlanCxt = pCxt}; - int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - return TSDB_CODE_SUCCESS; +static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { + pNode->pParent = pParent; + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { doSetLogicNodeParent((SLogicNode*)pChild, pNode); } +} + +static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } + +int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { + SLogicPlanContext cxt = {.pPlanCxt = pCxt}; + + SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pSubplan->id.queryId = pCxt->queryId; + pSubplan->id.groupId = 1; + pSubplan->id.subplanId = 1; + + int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode); + if (TSDB_CODE_SUCCESS == code) { + setLogicNodeParent(pSubplan->pNode); + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pSubplan->pNode)) { + pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; + } else { + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicSubplan = pSubplan; + } else { + nodesDestroyNode(pSubplan); + } + + return code; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8346aca76f..ae689c53d6 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -17,6 +17,7 @@ #include "functionMgt.h" #include "index.h" #include "planInt.h" +#include "ttime.h" #define OPTIMIZE_FLAG_MASK(n) (1 << n) @@ -32,8 +33,7 @@ typedef struct SOptimizeContext { bool optimized; } SOptimizeContext; -typedef int32_t (*FMatch)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); -typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); +typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan); typedef struct SOptimizeRule { char* pName; @@ -109,7 +109,6 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { } if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { return true; - // return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType); } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } @@ -231,9 +230,9 @@ static void setScanWindowInfo(SScanLogicNode* pScan) { } } -static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { +static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {0}; - int32_t code = osdMatch(pCxt, pLogicNode, &info); + int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { setScanWindowInfo((SScanLogicNode*)info.pScan); } @@ -635,8 +634,8 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) return code; } -static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - return cpdPushCondition(pCxt, pLogicNode); +static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + return cpdPushCondition(pCxt, pLogicSubplan->pNode); } static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { @@ -745,26 +744,310 @@ static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { return code; } -static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); +static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized); if (NULL == pSort) { return TSDB_CODE_SUCCESS; } return opkOptimizeImpl(pCxt, pSort); } -static const SOptimizeRule optimizeRuleSet[] = {{.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}}; +static bool smaOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent || + QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) || + WINDOW_TYPE_INTERVAL != ((SWindowLogicNode*)pNode->pParent)->winType) { + return false; + } + + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (0 == pScan->interval || NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) { + return false; + } + + return true; +} + +static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, SLogicNode** pOutput) { + SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); + if (NULL == pMerge) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pMerge->node.precision = pChild->precision; + pMerge->numOfChannels = 2; + pMerge->pMergeKeys = pMergeKeys; + pMerge->node.pTargets = pTargets; + pMerge->pInputs = nodesCloneList(pChild->pTargets); + if (NULL == pMerge->pInputs) { + nodesDestroyNode(pMerge); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pOutput = (SLogicNode*)pMerge; + return TSDB_CODE_SUCCESS; +} + +static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, + SLogicNode* pSmaScan) { + int32_t code = nodesListMakeAppend(&pMerge->pChildren, pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pMerge->pChildren, pSmaScan); + } + if (TSDB_CODE_SUCCESS == code) { + code = replaceLogicNode(pLogicSubplan, pInterval, pMerge); + pSmaScan->pParent = pMerge; + pInterval->pParent = pMerge; + } + return code; +} + +static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, + SLogicNode** pOutput) { + SScanLogicNode* pSmaScan = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); + if (NULL == pSmaScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pSmaScan->pScanCols = pCols; + pSmaScan->tableType = TSDB_SUPER_TABLE; + pSmaScan->tableId = pIndex->dstTbUid; + pSmaScan->stableId = pIndex->dstTbUid; + pSmaScan->scanType = SCAN_TYPE_TABLE; + pSmaScan->scanSeq[0] = pScan->scanSeq[0]; + pSmaScan->scanSeq[1] = pScan->scanSeq[1]; + pSmaScan->scanRange = pScan->scanRange; + pSmaScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; + + pSmaScan->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + pSmaScan->node.pTargets = nodesCloneList(pCols); + if (NULL == pSmaScan->pVgroupList || NULL == pSmaScan->node.pTargets) { + nodesDestroyNode(pSmaScan); + return TSDB_CODE_OUT_OF_MEMORY; + } + pSmaScan->pVgroupList->numOfVgroups = 1; + pSmaScan->pVgroupList->vgroups[0].vgId = pIndex->dstVgId; + memcpy(&(pSmaScan->pVgroupList->vgroups[0].epSet), &pIndex->epSet, sizeof(SEpSet)); + + *pOutput = (SLogicNode*)pSmaScan; + return TSDB_CODE_SUCCESS; +} + +static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { + if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit || + pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding || + pWindow->slidingUnit != pIndex->slidingUnit) { + return false; + } + if (IS_TSWINDOW_SPECIFIED(pScan->scanRange)) { + SInterval interval = {.interval = pIndex->interval, + .intervalUnit = pIndex->intervalUnit, + .offset = pIndex->offset, + .offsetUnit = TIME_UNIT_MILLISECOND, + .sliding = pIndex->sliding, + .slidingUnit = pIndex->slidingUnit, + .precision = pScan->node.precision}; + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + } + return true; +} + +static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->tableId = tableId; + pCol->tableType = TSDB_SUPER_TABLE; + pCol->colId = colId; + pCol->colType = COLUMN_TYPE_COLUMN; + strcpy(pCol->colName, ((SExprNode*)pFunc)->aliasName); + pCol->node.resType = ((SExprNode*)pFunc)->resType; + strcpy(pCol->node.aliasName, ((SExprNode*)pFunc)->aliasName); + return (SNode*)pCol; +} + +static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { + int32_t index = 0; + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pSmaFuncs) { + if (nodesEqualNode(pQueryFunc, pSmaFunc)) { + return index; + } + ++index; + } + return -1; +} + +static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, + int32_t* pWStrartIndex) { + SNodeList* pCols = NULL; + SNode* pFunc = NULL; + int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; + int32_t smaFuncIndex = -1; + *pWStrartIndex = -1; + FOREACH(pFunc, pFuncs) { + if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) { + *pWStrartIndex = index; + } + smaFuncIndex = smaOptFindSmaFunc(pFunc, pSmaFuncs); + if (smaFuncIndex < 0) { + break; + } else { + code = nodesListMakeStrictAppend(&pCols, smaOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + ++index; + } + + if (TSDB_CODE_SUCCESS == code && smaFuncIndex >= 0) { + *pOutput = pCols; + } else { + nodesDestroyList(pCols); + } + + return code; +} + +static int32_t smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, + int32_t* pWStrartIndex) { + SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; + if (!smaOptEqualInterval(pScan, pWindow, pIndex)) { + return TSDB_CODE_SUCCESS; + } + SNodeList* pSmaFuncs = NULL; + int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = smaOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + } + nodesDestroyList(pSmaFuncs); + return code; +} + +static SNode* smaOptCreateWStartTs() { + SFunctionNode* pWStart = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pWStart) { + return NULL; + } + strcpy(pWStart->functionName, "_wstartts"); + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) { + nodesDestroyNode(pWStart); + return NULL; + } + return (SNode*)pWStart; +} + +static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { + SOrderByExprNode* pMergeKey = nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pMergeKey) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pMergeKey->pExpr = nodesCloneNode(pCol); + if (NULL == pMergeKey->pExpr) { + nodesDestroyNode(pMergeKey); + return TSDB_CODE_OUT_OF_MEMORY; + } + pMergeKey->order = ORDER_ASC; + pMergeKey->nullOrder = NULL_ORDER_FIRST; + return nodesListMakeStrictAppend(pMergeKeys, pMergeKey); +} + +static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { + if (wstrartIndex < 0) { + SNode* pWStart = smaOptCreateWStartTs(); + if (NULL == pWStart) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pWStart); + return code; + } + wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; + } + return smaOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); +} + +static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { + SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; + SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); + if (NULL == pMergeTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SLogicNode* pSmaScan = NULL; + SLogicNode* pMerge = NULL; + SNodeList* pMergeKeys = NULL; + int32_t code = smaOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); + if (TSDB_CODE_SUCCESS == code) { + code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + } + if (TSDB_CODE_SUCCESS == code) { + code = smaOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); + } + if (TSDB_CODE_SUCCESS == code) { + code = smaOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); + } + return code; +} + +static int32_t smaOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { + SLogicNode* pSmaScan = NULL; + int32_t code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + if (TSDB_CODE_SUCCESS == code) { + code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan); + } + return code; +} + +static void smaOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } + +static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); + for (int32_t i = 0; i < nindexes; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); + SNodeList* pSmaCols = NULL; + int32_t wstrartIndex = -1; + code = smaOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { + code = smaOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); + taosArrayDestroyEx(pScan->pSmaIndexes, smaOptDestroySmaIndex); + pScan->pSmaIndexes = NULL; + break; + } + } + return code; +} + +static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaOptMayBeOptimized); + if (NULL == pScan) { + return TSDB_CODE_SUCCESS; + } + return smaOptimizeImpl(pCxt, pLogicSubplan, pScan); +} + +// clang-format off +static const SOptimizeRule optimizeRuleSet[] = { + {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, + {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, + {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaOptimize} +}; +// clang-format on static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule)); -static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { +static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { SOptimizeContext cxt = {.pPlanCxt = pCxt, .optimized = false}; do { cxt.optimized = false; for (int32_t i = 0; i < optimizeRuleNum; ++i) { - int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicNode); + int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicSubplan); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -773,4 +1056,6 @@ static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { return TSDB_CODE_SUCCESS; } -int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { return applyOptimizeRule(pCxt, pLogicNode); } +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + return applyOptimizeRule(pCxt, pLogicSubplan); +} diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 21555e66a2..f33856c4ce 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -80,29 +80,12 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE return TSDB_CODE_SUCCESS; } -static int32_t splReplaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { - if (NULL == pOld->pParent) { - pSubplan->pNode = (SLogicNode*)pNew; - return TSDB_CODE_SUCCESS; - } - - SNode* pNode; - FOREACH(pNode, pOld->pParent->pChildren) { - if (nodesEqualNode(pNode, pOld)) { - REPLACE_NODE(pNew); - pNew->pParent = pOld->pParent; - return TSDB_CODE_SUCCESS; - } - } - return TSDB_CODE_PLAN_INTERNAL_ERROR; -} - static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, ESubplanType subplanType) { SExchangeLogicNode* pExchange = NULL; int32_t code = splCreateExchangeNode(pCxt, pSplitNode, &pExchange); if (TSDB_CODE_SUCCESS == code) { - code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange); + code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange); } if (TSDB_CODE_SUCCESS == code) { pSubplan->subplanType = subplanType; @@ -282,7 +265,7 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic code = stbSplAppendWStart(pPartWin->pFuncs, &index); } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pPartWin->pFuncs, &pPartWin->node.pTargets); + code = createColumnByRewriteExprs(pPartWin->pFuncs, &pPartWin->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { nodesDestroyNode(pMergeWindow->pTspk); @@ -328,7 +311,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla if (NULL == pSubplan) { code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); } else { - code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); + code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); } } if (TSDB_CODE_SUCCESS != code) { @@ -442,7 +425,7 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; - code = createColumnByRewriteExps(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); + code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { pMergeAgg->pGroupKeys = nodesCloneList(pPartAgg->node.pTargets); @@ -457,7 +440,7 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO code = stbSplRewriteFuns(pFunc, &pPartAgg->pAggFuncs, &pMergeAgg->pAggFuncs); } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pPartAgg->pAggFuncs, &pPartAgg->node.pTargets); + code = createColumnByRewriteExprs(pPartAgg->pAggFuncs, &pPartAgg->node.pTargets); } nodesDestroyList(pFunc); @@ -897,12 +880,56 @@ static int32_t unionDistinctSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) return code; } +typedef struct SSmaIndexSplitInfo { + SMergeLogicNode* pMerge; + SLogicSubplan* pSubplan; +} SSmaIndexSplitInfo; + +static SLogicNode* smaIdxSplMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_MERGE == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = smaIdxSplMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static bool smaIdxSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SSmaIndexSplitInfo* pInfo) { + SLogicNode* pSplitNode = smaIdxSplMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pMerge = (SMergeLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + +static int32_t smaIndexSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SSmaIndexSplitInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)smaIdxSplFindSplitNode, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pMerge); + if (TSDB_CODE_SUCCESS == code) { + info.pMerge->srcGroupId = pCxt->groupId; + } + ++(pCxt->groupId); + pCxt->split = true; + return code; +} + // clang-format off static const SSplitRule splitRuleSet[] = { {.pName = "SuperTableSplit", .splitFunc = stableSplit}, {.pName = "SingleTableJoinSplit", .splitFunc = singleTableJoinSplit}, {.pName = "UnionAllSplit", .splitFunc = unionAllSplit}, - {.pName = "UnionDistinctSplit", .splitFunc = unionDistinctSplit} + {.pName = "UnionDistinctSplit", .splitFunc = unionDistinctSplit}, + {.pName = "SmaIndexSplit", .splitFunc = smaIndexSplit} }; // clang-format on @@ -936,14 +963,6 @@ static int32_t applySplitRule(SPlanContext* pCxt, SLogicSubplan* pSubplan) { return TSDB_CODE_SUCCESS; } -static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { - pNode->pParent = pParent; - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { doSetLogicNodeParent((SLogicNode*)pChild, pNode); } -} - -static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } - static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { TSWAP(((SScanLogicNode*)pNode)->pVgroupList, pSubplan->pVgroupList); @@ -954,37 +973,10 @@ static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { FOREACH(pChild, pNode->pChildren) { setVgroupsInfo((SLogicNode*)pChild, pSubplan); } } -int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { - SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - if (NULL == pSubplan) { - return TSDB_CODE_OUT_OF_MEMORY; +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicSubplan->pNode)) { + setVgroupsInfo(pLogicSubplan->pNode, pLogicSubplan); + return TSDB_CODE_SUCCESS; } - - pSubplan->pNode = nodesCloneNode(pLogicNode); - if (NULL == pSubplan->pNode) { - nodesDestroyNode(pSubplan); - return TSDB_CODE_OUT_OF_MEMORY; - } - - pSubplan->id.queryId = pCxt->queryId; - pSubplan->id.groupId = 1; - setLogicNodeParent(pSubplan->pNode); - - int32_t code = TSDB_CODE_SUCCESS; - if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicNode)) { - pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifyLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifyLogicNode*)pSubplan->pNode)->pDataBlocks); - setVgroupsInfo(pSubplan->pNode, pSubplan); - } else { - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; - code = applySplitRule(pCxt, pSubplan); - } - - if (TSDB_CODE_SUCCESS == code) { - *pLogicSubplan = pSubplan; - } else { - nodesDestroyNode(pSubplan); - } - - return code; -} \ No newline at end of file + return applySplitRule(pCxt, pLogicSubplan); +} diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 63d31912f0..232a49ee3a 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -69,7 +69,7 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList) { +int32_t createColumnByRewriteExprs(SNodeList* pExprs, SNodeList** pList) { SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; if (NULL == cxt.pList) { return TSDB_CODE_OUT_OF_MEMORY; @@ -85,3 +85,37 @@ int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList) { } return cxt.errCode; } + +int32_t createColumnByRewriteExpr(SNode* pExpr, SNodeList** pList) { + SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; + if (NULL == cxt.pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + nodesWalkExpr(pExpr, doCreateColumn, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pList); + return cxt.errCode; + } + if (NULL == *pList) { + *pList = cxt.pList; + } + return cxt.errCode; +} + +int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { + if (NULL == pOld->pParent) { + pSubplan->pNode = (SLogicNode*)pNew; + return TSDB_CODE_SUCCESS; + } + + SNode* pNode; + FOREACH(pNode, pOld->pParent->pChildren) { + if (nodesEqualNode(pNode, pOld)) { + REPLACE_NODE(pNew); + pNew->pParent = pOld->pParent; + return TSDB_CODE_SUCCESS; + } + } + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 1921b16388..83657d27d0 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -26,16 +26,15 @@ static void dumpQueryPlan(SQueryPlan* pPlan) { } int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) { - SLogicNode* pLogicNode = NULL; SLogicSubplan* pLogicSubplan = NULL; SQueryLogicPlan* pLogicPlan = NULL; - int32_t code = createLogicPlan(pCxt, &pLogicNode); + int32_t code = createLogicPlan(pCxt, &pLogicSubplan); if (TSDB_CODE_SUCCESS == code) { - code = optimizeLogicPlan(pCxt, pLogicNode); + code = optimizeLogicPlan(pCxt, pLogicSubplan); } if (TSDB_CODE_SUCCESS == code) { - code = splitLogicPlan(pCxt, pLogicNode, &pLogicSubplan); + code = splitLogicPlan(pCxt, pLogicSubplan); } if (TSDB_CODE_SUCCESS == code) { code = scaleOutLogicPlan(pCxt, pLogicSubplan, &pLogicPlan); @@ -47,7 +46,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo dumpQueryPlan(*pPlan); } - nodesDestroyNode(pLogicNode); nodesDestroyNode(pLogicSubplan); nodesDestroyNode(pLogicPlan); terrno = code; diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index d5d37fda64..4bfb9a6fda 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -15,6 +15,7 @@ #include "planTestUtil.h" #include "planner.h" +#include "tglobal.h" using namespace std; @@ -42,7 +43,17 @@ TEST_F(PlanOtherTest, createStreamUseSTable) { TEST_F(PlanOtherTest, createSmaIndex) { useDb("root", "test"); - run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) interval(10s)"); + run("CREATE SMA INDEX idx1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); + + run("SELECT SUM(c4) FROM t1 INTERVAL(10s)"); + + run("SELECT _WSTARTTS, MIN(c3 + 10) FROM t1 " + "WHERE ts BETWEEN TIMESTAMP '2022-04-01 00:00:00' AND TIMESTAMP '2022-04-30 23:59:59.999' INTERVAL(10s)"); + + run("SELECT SUM(c4), MAX(c3) FROM t1 INTERVAL(10s)"); + + tsQuerySmaOptimize = 0; + run("SELECT SUM(c4) FROM t1 INTERVAL(10s)"); } TEST_F(PlanOtherTest, explain) { diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index f82e10e998..7d1ac84aea 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -28,6 +28,12 @@ TEST_F(PlanSubqeuryTest, basic) { run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); run("SELECT c1 FROM (SELECT TODAY() AS c1 FROM t1)"); + + run("SELECT NOW() FROM t1"); + + run("SELECT NOW() FROM (SELECT * FROM t1)"); + + // run("SELECT NOW() FROM (SELECT * FROM t1) ORDER BY ts"); } TEST_F(PlanSubqeuryTest, doubleGroupBy) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index f5c8b58e43..57d7cb6608 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -14,12 +14,14 @@ */ #include "planTestUtil.h" + #include #include #include #include "cmdnodes.h" +#include "mockCatalogService.h" #include "parser.h" #include "planInt.h" @@ -104,13 +106,12 @@ class PlannerTestBaseImpl { SPlanContext cxt = {0}; setPlanContext(pQuery, &cxt); - SLogicNode* pLogicNode = nullptr; - doCreateLogicPlan(&cxt, &pLogicNode); - - doOptimizeLogicPlan(&cxt, pLogicNode); - SLogicSubplan* pLogicSubplan = nullptr; - doSplitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + doCreateLogicPlan(&cxt, &pLogicSubplan); + + doOptimizeLogicPlan(&cxt, pLogicSubplan); + + doSplitLogicPlan(&cxt, pLogicSubplan); SQueryLogicPlan* pLogicPlan = nullptr; doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); @@ -164,13 +165,12 @@ class PlannerTestBaseImpl { SPlanContext cxt = {0}; setPlanContext(stmtEnv_.pQuery_, &cxt); - SLogicNode* pLogicNode = nullptr; - doCreateLogicPlan(&cxt, &pLogicNode); - - doOptimizeLogicPlan(&cxt, pLogicNode); - SLogicSubplan* pLogicSubplan = nullptr; - doSplitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + doCreateLogicPlan(&cxt, &pLogicSubplan); + + doOptimizeLogicPlan(&cxt, pLogicSubplan); + + doSplitLogicPlan(&cxt, pLogicSubplan); SQueryLogicPlan* pLogicPlan = nullptr; doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); @@ -324,19 +324,19 @@ class PlannerTestBaseImpl { res_.ast_ = toString(pQuery->pRoot); } - void doCreateLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { - DO_WITH_THROW(createLogicPlan, pCxt, pLogicNode); - res_.rawLogicPlan_ = toString((SNode*)(*pLogicNode)); + void doCreateLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { + DO_WITH_THROW(createLogicPlan, pCxt, pLogicSubplan); + res_.rawLogicPlan_ = toString((SNode*)(*pLogicSubplan)); } - void doOptimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { - DO_WITH_THROW(optimizeLogicPlan, pCxt, pLogicNode); - res_.optimizedLogicPlan_ = toString((SNode*)pLogicNode); + void doOptimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + DO_WITH_THROW(optimizeLogicPlan, pCxt, pLogicSubplan); + res_.optimizedLogicPlan_ = toString((SNode*)pLogicSubplan); } - void doSplitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { - DO_WITH_THROW(splitLogicPlan, pCxt, pLogicNode, pLogicSubplan); - res_.splitLogicPlan_ = toString((SNode*)(*pLogicSubplan)); + void doSplitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + DO_WITH_THROW(splitLogicPlan, pCxt, pLogicSubplan); + res_.splitLogicPlan_ = toString((SNode*)(pLogicSubplan)); } void doScaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan) { @@ -363,6 +363,7 @@ class PlannerTestBaseImpl { } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { SMCreateSmaReq req = {0}; tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); + g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); pCxt->streamQuery = true; } else if (QUERY_NODE_CREATE_STREAM_STMT == nodeType(pQuery->pRoot)) { diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 069595390d..d3828d9ee4 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -110,9 +110,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDisp return 0; } // continue dispatch - if (pTask->dispatchType != TASK_DISPATCH__NONE) { - streamDispatch(pTask, pMsgCb); - } + streamDispatch(pTask, pMsgCb); return 0; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 6fb5d75d86..59ec2b5ceb 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -100,7 +100,6 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM .upstreamNodeId = pTask->nodeId, .blockNum = blockNum, }; - qInfo("dispatch from task %d (child id %d)", pTask->taskId, pTask->childId); req.data = taosArrayInit(blockNum, sizeof(void*)); req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); @@ -142,11 +141,14 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM break; } } - ASSERT(vgId != 0); } + ASSERT(vgId != 0); req.taskId = downstreamTaskId; + qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->childId, + downstreamTaskId, vgId); + // serialize int32_t tlen; tEncodeSize(tEncodeStreamDispatchReq, &req, tlen, code); @@ -180,6 +182,7 @@ FAIL: } int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) { + ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); #if 1 int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 10218f69e6..e7777af749 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -159,7 +159,7 @@ typedef struct SSyncNode { SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; SSyncSnapshotReceiver* pNewNodeReceiver; - SSnapshotMeta sMeta; + // SSnapshotMeta sMeta; } SSyncNode; @@ -194,15 +194,15 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDrop); +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex, bool* isDrop); SSyncNode* syncNodeAcquire(int64_t rid); void syncNodeRelease(SSyncNode* pNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); -void syncNodeBecomeFollower(SSyncNode* pSyncNode); -void syncNodeBecomeLeader(SSyncNode* pSyncNode); +void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr); +void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr); void syncNodeCandidate2Leader(SSyncNode* pSyncNode); void syncNodeFollower2Candidate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 86c5fab87c..e72e1e7be7 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -35,6 +35,7 @@ typedef struct SRaftCfg { char path[TSDB_FILENAME_LEN * 2]; int8_t isStandBy; int8_t snapshotEnable; + SyncIndex lastConfigIndex; } SRaftCfg; SRaftCfg *raftCfgOpen(const char *path); @@ -52,8 +53,9 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); typedef struct SRaftCfgMeta { - int8_t isStandBy; - int8_t snapshotEnable; + int8_t isStandBy; + int8_t snapshotEnable; + SyncIndex lastConfigIndex; } SRaftCfgMeta; int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 9f03ac3e55..e0cbcf0744 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -49,8 +49,8 @@ void raftStoreClearVote(SRaftStore *pRaftStore); void raftStoreNextTerm(SRaftStore *pRaftStore); void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson); -cJSON *raftStore2Json(SRaftStore *pRaftStore); -char *raftStore2Str(SRaftStore *pRaftStore); +cJSON * raftStore2Json(SRaftStore *pRaftStore); +char * raftStore2Str(SRaftStore *pRaftStore); // for debug ------------------- void raftStorePrint(SRaftStore *pObj); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index b16e47b51e..a6170a92e3 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,10 +39,11 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void *pReader; - void *pCurrentBlock; + void * pReader; + void * pCurrentBlock; int32_t blockLen; SSnapshot snapshot; + SSyncCfg lastConfig; int64_t sendingMS; SSyncNode *pSyncNode; int32_t replicaIndex; @@ -58,28 +59,29 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); +char * snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void *pWriter; + void * pWriter; SyncTerm term; SyncTerm privateTerm; SSyncNode *pSyncNode; - int32_t replicaIndex; + SRaftId fromId; + } SSyncSnapshotReceiver; -SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex); +SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SRaftId fromId); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 3c558b60c8..29ee263756 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -88,6 +88,7 @@ // /\ UNCHANGED <> // /\ UNCHANGED <> // + int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; @@ -150,7 +151,245 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { "ths->state:%d, logOK:%d", pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); - syncNodeBecomeFollower(ths); + syncNodeBecomeFollower(ths, "from candidate by append entries"); + + // ret or reply? + return ret; + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + // preIndex = -1, or has preIndex entry in local log + assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + sTrace( + "syncNodeOnAppendEntriesCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); + + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); + assert(pExtraEntry != NULL); + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // log not match, conflict + assert(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); + assert(pRollBackEntry != NULL); + + // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta; + cbMeta.index = pRollBackEntry->index; + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pAppendEntry->index; + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); + + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pAppendEntry->index; + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 3; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else { + assert(0); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + // maybe update commit index from leader + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index + ths->commitIndex = pMsg->commitIndex; + + // call back Wal + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + + int32_t code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); + } + } + } + + return ret; +} + +#if 0 +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesLog2(logBuf, pMsg); + + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + // reset elect timer + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + assert(pMsg->dataLen >= 0); + + SyncTerm localPreLogTerm = 0; + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); + assert(pEntry != NULL); + localPreLogTerm = pEntry->term; + syncEntryDestory(pEntry); + } + + bool logOK = + (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); + + // reject request + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + sTrace( + "syncNodeOnAppendEntriesCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + sTrace( + "syncNodeOnAppendEntriesCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + + syncNodeBecomeFollower(ths, "from candidate by append entries"); // ret or reply? return ret; @@ -375,22 +614,24 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // I am in newConfig if (hit) { - syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop); + syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); // change isStandBy to normal if (!isDrop) { if (ths->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(ths); + syncNodeBecomeLeader(ths, "config change"); } else { - syncNodeBecomeFollower(ths); + syncNodeBecomeFollower(ths, "config change"); } } - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); + if (gRaftDetailLog) { + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); + taosMemoryFree(sOld); + taosMemoryFree(sNew); + } } // always call FpReConfigCb @@ -399,10 +640,12 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { cbMeta.currentTerm = ths->pRaftStore->currentTerm; cbMeta.index = pEntry->index; cbMeta.term = pEntry->term; + cbMeta.newCfg = newSyncCfg; cbMeta.oldCfg = oldSyncCfg; + cbMeta.seqNum = pEntry->seqNum; cbMeta.flag = 0x11; cbMeta.isDrop = isDrop; - ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta); + ths->pFsm->FpReConfigCb(ths->pFsm, &rpcMsg, cbMeta); } } @@ -433,6 +676,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } +#endif static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t code; @@ -469,7 +713,7 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - sInfo("sync event log truncate, from %ld to %ld", delBegin, delEnd); + sDebug("vgId:%d sync event log truncate, from %ld to %ld", ths->vgId, delBegin, delEnd); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); return code; @@ -571,7 +815,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs if (condition) { sTrace("recv SyncAppendEntries, candidate to follower"); - syncNodeBecomeFollower(ths); + syncNodeBecomeFollower(ths, "from candidate by append entries"); // do not reply? return ret; } @@ -742,6 +986,18 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs if (pMsg->commitIndex > ths->commitIndex) { // has commit entry in local if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { + // advance commit index to sanpshot first + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + if (snapshot.lastApplyIndex >= 0 && snapshot.lastApplyIndex > ths->commitIndex) { + SyncIndex commitBegin = ths->commitIndex; + SyncIndex commitEnd = snapshot.lastApplyIndex; + ths->commitIndex = snapshot.lastApplyIndex; + + sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", ths->vgId, commitBegin, + commitEnd, syncUtilState2String(ths->state)); + } + SyncIndex beginIndex = ths->commitIndex + 1; SyncIndex endIndex = pMsg->commitIndex; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 5a543e1605..0e116e13ee 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -121,7 +121,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex); - { + if (gRaftDetailLog) { SSnapshot snapshot; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", @@ -147,7 +147,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (pMsg->success) { // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); - sTrace("update next match, index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); + + if (gRaftDetailLog) { + sTrace("update next match, index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); + } // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); @@ -159,7 +162,9 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); - sTrace("update next not match, begin, index:%ld, success:%d", nextIndex, pMsg->success); + if (gRaftDetailLog) { + sTrace("update next index not match, begin, index:%ld, success:%d", nextIndex, pMsg->success); + } // notice! int64, uint64 if (nextIndex > SYNC_INDEX_BEGIN) { @@ -178,9 +183,26 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries pMsg->privateTerm < pSender->privateTerm) { snapshotSenderStart(pSender); - char* s = snapshotSender2Str(pSender); - sInfo("sync event snapshot send start sender first time, sender:%s", s); - taosMemoryFree(s); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + + if (gRaftDetailLog) { + char* s = snapshotSender2Str(pSender); + sDebug( + "vgId:%d sync event snapshot send to %s:%d start sender first time, lastApplyIndex:%ld lastApplyTerm:%lu " + "lastConfigIndex:%ld" + "sender:%s", + ths->vgId, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, + pSender->snapshot.lastConfigIndex, s); + taosMemoryFree(s); + } else { + sDebug( + "vgId:%d sync event snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " + "lastApplyTerm:%lu lastConfigIndex:%ld", + ths->vgId, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, + pSender->snapshot.lastConfigIndex); + } } SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; @@ -195,12 +217,14 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); - sTrace("update next not match, end, index:%ld, success:%d", nextIndex, pMsg->success); + if (gRaftDetailLog) { + sTrace("update next index not match, end, index:%ld, success:%d", nextIndex, pMsg->success); + } } syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); - { + if (gRaftDetailLog) { SSnapshot snapshot; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index c092b31adf..eac9c1fbd8 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -48,13 +48,28 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { syncIndexMgrLog2("==syncNodeMaybeAdvanceCommitIndex== pNextIndex", pSyncNode->pNextIndex); syncIndexMgrLog2("==syncNodeMaybeAdvanceCommitIndex== pMatchIndex", pSyncNode->pMatchIndex); + // advance commit index to sanpshot first + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex > 0 && snapshot.lastApplyIndex > pSyncNode->commitIndex) { + SyncIndex commitBegin = pSyncNode->commitIndex; + SyncIndex commitEnd = snapshot.lastApplyIndex; + pSyncNode->commitIndex = snapshot.lastApplyIndex; + + sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", pSyncNode->vgId, + pSyncNode->commitIndex, snapshot.lastApplyIndex, syncUtilState2String(pSyncNode->state)); + } + // update commit index SyncIndex newCommitIndex = pSyncNode->commitIndex; - for (SyncIndex index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); index > pSyncNode->commitIndex; - --index) { + for (SyncIndex index = syncNodeGetLastIndex(pSyncNode); index > pSyncNode->commitIndex; --index) { bool agree = syncAgree(pSyncNode, index); - sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index, - pSyncNode->commitIndex); + + if (gRaftDetailLog) { + sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index, + pSyncNode->commitIndex); + } + if (agree) { // term SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); @@ -64,16 +79,21 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { // update commit index newCommitIndex = index; - sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%ld commit, pSyncNode->commitIndex:%ld", - newCommitIndex, pSyncNode->commitIndex); + + if (gRaftDetailLog) { + sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%ld commit, pSyncNode->commitIndex:%ld", + newCommitIndex, pSyncNode->commitIndex); + } syncEntryDestory(pEntry); break; } else { - sTrace( - "syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%lu, " - "pSyncNode->pRaftStore->currentTerm:%lu", - pEntry->term, pSyncNode->pRaftStore->currentTerm); + if (gRaftDetailLog) { + sTrace( + "syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%lu, " + "pSyncNode->pRaftStore->currentTerm:%lu", + pEntry->term, pSyncNode->pRaftStore->currentTerm); + } } syncEntryDestory(pEntry); @@ -84,7 +104,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex beginIndex = pSyncNode->commitIndex + 1; SyncIndex endIndex = newCommitIndex; - sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex); + if (gRaftDetailLog) { + sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex); + } // update commit index pSyncNode->commitIndex = newCommitIndex; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 945d59646b..e67439f8fe 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -40,7 +40,7 @@ int32_t syncEnvStart() { // gSyncEnv = doSyncEnvStart(gSyncEnv); gSyncEnv = doSyncEnvStart(); assert(gSyncEnv != NULL); - sTrace("syncEnvStart ok!"); + sTrace("sync env start ok"); return ret; } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index ecc1c8f1e2..18cb55b417 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -119,7 +119,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9516df64da..4d410644bb 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -87,7 +87,9 @@ int64_t syncOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); assert(pSyncNode != NULL); - syncNodeLog2("syncNodeOpen open success", pSyncNode); + if (gRaftDetailLog) { + syncNodeLog2("syncNodeOpen open success", pSyncNode); + } pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode); if (pSyncNode->rid < 0) { @@ -173,20 +175,105 @@ int32_t syncSetStandby(int64_t rid) { int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { int32_t ret = 0; - char* configChange = syncCfg2Str((SSyncCfg*)pSyncCfg); - sInfo("==syncReconfig== newconfig:%s", configChange); + char* newconfig = syncCfg2Str((SSyncCfg*)pSyncCfg); + + if (gRaftDetailLog) { + sInfo("==syncReconfig== newconfig:%s", newconfig); + } SRpcMsg rpcMsg = {0}; rpcMsg.msgType = TDMT_SYNC_CONFIG_CHANGE; rpcMsg.info.noResp = 1; - rpcMsg.contLen = strlen(configChange) + 1; + rpcMsg.contLen = strlen(newconfig) + 1; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", configChange); - taosMemoryFree(configChange); + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", newconfig); + taosMemoryFree(newconfig); ret = syncPropose(rid, &rpcMsg, false); return ret; } +int32_t syncLeaderTransfer(int64_t rid) { + int32_t ret = 0; + + return ret; +} + +int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return false; + } + assert(rid == pSyncNode->rid); + int32_t ret = 0; + + if (pSyncNode->replicaNum == 1) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + sError("only one replica, cannot drop leader"); + return TAOS_SYNC_ONLY_ONE_REPLICA; + } + + SyncLeaderTransfer* pMsg = syncLeaderTransferBuild(pSyncNode->vgId); + pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort); + pMsg->newLeaderId.vgId = pSyncNode->vgId; + ASSERT(pMsg != NULL); + SRpcMsg rpcMsg = {0}; + syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); + syncLeaderTransferDestroy(pMsg); + + ret = syncPropose(rid, &rpcMsg, false); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +int32_t syncReconfigRaw(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { + int32_t ret = 0; + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); + + pRpcMsg->msgType = TDMT_SYNC_CONFIG_CHANGE; + pRpcMsg->info.noResp = 1; + pRpcMsg->contLen = strlen(newconfig) + 1; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + snprintf(pRpcMsg->pCont, pRpcMsg->contLen, "%s", newconfig); + taosMemoryFree(newconfig); + + return ret; +} + +bool syncCanLeaderTransfer(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return false; + } + assert(rid == pSyncNode->rid); + + if (pSyncNode->replicaNum == 1) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return false; + } + + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return true; + } + + bool matchOK = true; + if (pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE || pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + SyncIndex myCommitIndex = pSyncNode->commitIndex; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SyncIndex peerMatchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId)[i]); + if (peerMatchIndex < myCommitIndex) { + matchOK = false; + } + } + } + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return matchOK; +} + +int32_t syncGiveUpLeader(int64_t rid) { return 0; } + int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = syncPropose(rid, pMsg, isWeak); return ret; @@ -222,7 +309,9 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { return -1; } assert(rid == pSyncNode->rid); - *sMeta = pSyncNode->sMeta; + sMeta->lastConfigIndex = pSyncNode->pRaftCfg->lastConfigIndex; + + sTrace("sync get snapshot meta: lastConfigIndex:%ld", pSyncNode->pRaftCfg->lastConfigIndex); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return 0; @@ -374,13 +463,14 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - sTrace("syncPropose msgType:%d ", pMsg->msgType); + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; - int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) return TAOS_SYNC_PROPOSE_OTHER_ERROR; - + if (pSyncNode == NULL) { + return TAOS_SYNC_PROPOSE_OTHER_ERROR; + } assert(rid == pSyncNode->rid); + sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -411,6 +501,8 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; + sDebug("vgId:%d sync event sync open", pSyncInfo->vgId); + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); @@ -430,6 +522,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { SRaftCfgMeta meta; meta.isStandBy = pSyncInfo->isStandBy; meta.snapshotEnable = pSyncInfo->snapshotEnable; + meta.lastConfigIndex = SYNC_INDEX_INVALID; ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath); assert(ret == 0); @@ -439,9 +532,11 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { assert(pSyncNode->pRaftCfg != NULL); pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; - char* seralized = raftCfg2Str(pSyncNode->pRaftCfg); - sInfo("syncNodeOpen update config :%s", seralized); - taosMemoryFree(seralized); + if (gRaftDetailLog) { + char* seralized = raftCfg2Str(pSyncNode->pRaftCfg); + sInfo("syncNodeOpen update config :%s", seralized); + taosMemoryFree(seralized); + } raftCfgClose(pSyncNode->pRaftCfg); } @@ -612,14 +707,14 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { } // snapshot receivers - pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, 100); + pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, EMPTY_RAFT_ID); // start in syncNodeStart // start raft // syncNodeBecomeFollower(pSyncNode); // snapshot meta - pSyncNode->sMeta.lastConfigIndex = -1; + // pSyncNode->sMeta.lastConfigIndex = -1; return pSyncNode; } @@ -628,9 +723,7 @@ void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { raftStoreNextTerm(pSyncNode->pRaftStore); - syncNodeBecomeLeader(pSyncNode); - - syncNodeLog2("==state change become leader immediately==", pSyncNode); + syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms @@ -638,41 +731,22 @@ void syncNodeStart(SSyncNode* pSyncNode) { syncNodeAppendNoop(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica - /* - sInfo("==syncNodeStart== RestoreFinish begin 1 replica tsem_wait %p", pSyncNode); - tsem_wait(&pSyncNode->restoreSem); - sInfo("==syncNodeStart== RestoreFinish end 1 replica tsem_wait %p", pSyncNode); - */ - - /* - while (pSyncNode->restoreFinish != true) { - taosMsleep(10); + if (gRaftDetailLog) { + syncNodeLog2("==state change become leader immediately==", pSyncNode); } - */ - sInfo("==syncNodeStart== restoreFinish ok 1 replica %p vgId:%d", pSyncNode, pSyncNode->vgId); return; } - syncNodeBecomeFollower(pSyncNode); + syncNodeBecomeFollower(pSyncNode, "first start"); - // for test - int32_t ret = 0; + // int32_t ret = 0; // ret = syncNodeStartPingTimer(pSyncNode); - assert(ret == 0); + // assert(ret == 0); - /* - sInfo("==syncNodeStart== RestoreFinish begin multi replica tsem_wait %p", pSyncNode); - tsem_wait(&pSyncNode->restoreSem); - sInfo("==syncNodeStart== RestoreFinish end multi replica tsem_wait %p", pSyncNode); - */ - - /* - while (pSyncNode->restoreFinish != true) { - taosMsleep(10); + if (gRaftDetailLog) { + syncNodeLog2("==state change become leader immediately==", pSyncNode); } - */ - sInfo("==syncNodeStart== restoreFinish ok multi replica %p vgId:%d", pSyncNode, pSyncNode->vgId); } void syncNodeStartStandBy(SSyncNode* pSyncNode) { @@ -687,6 +761,8 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodeClose(SSyncNode* pSyncNode) { + sDebug("vgId:%d sync event sync close", pSyncNode->vgId); + int32_t ret; assert(pSyncNode != NULL); @@ -1061,7 +1137,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { int len = 256; char* s = (char*)taosMemoryMalloc(len); snprintf(s, len, - "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " + "syncNode: vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " "electTimerLogicClock:%lu, " "electTimerLogicClockUser:%lu, " "electTimerMS:%d, replicaNum:%d", @@ -1071,9 +1147,11 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { return s; } -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDrop) { +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex, bool* isDrop) { SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg; pSyncNode->pRaftCfg->cfg = *newConfig; + pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex; + int32_t ret = 0; // init internal @@ -1106,13 +1184,12 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDro pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); - // isDrop - *isDrop = true; - bool IamInOld, IamInNew; + bool IamInOld = false; + bool IamInNew = false; for (int i = 0; i < oldConfig.replicaNum; ++i) { if (strcmp((oldConfig.nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && (oldConfig.nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { - *isDrop = false; + IamInOld = true; break; } } @@ -1120,18 +1197,26 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDro for (int i = 0; i < newConfig->replicaNum; ++i) { if (strcmp((newConfig->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && (newConfig->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { - *isDrop = false; + IamInNew = true; break; } } - if (!(*isDrop)) { - // change isStandBy to normal - pSyncNode->pRaftCfg->isStandBy = 0; + *isDrop = true; + if (IamInOld && !IamInNew) { + *isDrop = true; + } else { + *isDrop = false; } + if (IamInNew) { + pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal + } raftCfgPersist(pSyncNode->pRaftCfg); - syncNodeLog2("==syncNodeUpdateConfig==", pSyncNode); + + if (gRaftDetailLog) { + syncNodeLog2("==syncNodeUpdateConfig==", pSyncNode); + } } SSyncNode* syncNodeAcquire(int64_t rid) { @@ -1149,12 +1234,15 @@ void syncNodeRelease(SSyncNode* pNode) { taosReleaseRef(tsNodeRefId, pNode->rid) void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, term); - syncNodeBecomeFollower(pSyncNode); + syncNodeBecomeFollower(pSyncNode, "update term"); raftStoreClearVote(pSyncNode->pRaftStore); } } -void syncNodeBecomeFollower(SSyncNode* pSyncNode) { +void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { + sDebug("vgId:%d sync event become follower, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, + debugStr); + // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { pSyncNode->leaderCache = EMPTY_RAFT_ID; @@ -1186,7 +1274,10 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode) { // evoterLog |-> voterLog[i]]} // /\ UNCHANGED <> // -void syncNodeBecomeLeader(SSyncNode* pSyncNode) { +void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { + sDebug("vgId:%d sync event become leader, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, + debugStr); + // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -1237,7 +1328,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) { void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); assert(voteGrantedMajority(pSyncNode->pVotesGranted)); - syncNodeBecomeLeader(pSyncNode); + syncNodeBecomeLeader(pSyncNode, "candidate to leader"); syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); @@ -1260,14 +1351,14 @@ void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { void syncNodeLeader2Follower(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); - syncNodeBecomeFollower(pSyncNode); + syncNodeBecomeFollower(pSyncNode, "leader to follower"); syncNodeLog2("==state change syncNodeLeader2Follower==", pSyncNode); } void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - syncNodeBecomeFollower(pSyncNode); + syncNodeBecomeFollower(pSyncNode, "candidate to follower"); syncNodeLog2("==state change syncNodeCandidate2Follower==", pSyncNode); } @@ -1467,9 +1558,11 @@ void syncNodeLog(SSyncNode* pObj) { } void syncNodeLog2(char* s, SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - sTraceLong("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncNode2Str(pObj); + sTraceLong("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ------ local funciton --------- @@ -1721,20 +1814,78 @@ const char* syncStr(ESyncState state) { } } +static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { + SyncLeaderTransfer* pSyncLeaderTransfer; + if (syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId))) { + } + + return 0; +} + +static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { + SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; + + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(pRpcMsg->pCont, &newSyncCfg); + ASSERT(ret == 0); + + // update new config myIndex + bool IamInNew = false; + for (int i = 0; i < newSyncCfg.replicaNum; ++i) { + if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && + ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { + newSyncCfg.myIndex = i; + IamInNew = true; + break; + } + } + + bool isDrop; + + if (IamInNew || (!IamInNew && ths->state != TAOS_SYNC_STATE_LEADER)) { + syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); + + // change isStandBy to normal + if (!isDrop) { + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(ths, "config change"); + } else { + syncNodeBecomeFollower(ths, "config change"); + } + } + + if (gRaftDetailLog) { + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); + taosMemoryFree(sOld); + taosMemoryFree(sNew); + } + } + + // always call FpReConfigCb + if (ths->pFsm->FpReConfigCb != NULL) { + SReConfigCbMeta cbMeta = {0}; + cbMeta.code = 0; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.index = pEntry->index; + cbMeta.term = pEntry->term; + cbMeta.newCfg = newSyncCfg; + cbMeta.oldCfg = oldSyncCfg; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.flag = 0x11; + cbMeta.isDrop = isDrop; + ths->pFsm->FpReConfigCb(ths->pFsm, pRpcMsg, cbMeta); + } + + return 0; +} + int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; - sInfo("sync event commit from index:%" PRId64 " to index:%" PRId64 ", %s", beginIndex, endIndex, - syncUtilState2String(state)); - - // maybe execute by leader, skip snapshot - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (ths->pFsm->FpGetSnapshot != NULL) { - ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - } - if (beginIndex <= snapshot.lastApplyIndex) { - beginIndex = snapshot.lastApplyIndex + 1; - } + sDebug("vgId:%d sync event commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, beginIndex, + endIndex, syncUtilState2String(state)); // execute fsm if (ths->pFsm != NULL) { @@ -1748,6 +1899,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); + // user commit if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -1764,57 +1916,14 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, // config change if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) { - SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; + code = syncNodeConfigChange(ths, &rpcMsg, pEntry); + ASSERT(code == 0); + } - SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); - ASSERT(ret == 0); - - // update new config myIndex - bool hit = false; - for (int i = 0; i < newSyncCfg.replicaNum; ++i) { - if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && - ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { - newSyncCfg.myIndex = i; - hit = true; - break; - } - } - - SReConfigCbMeta cbMeta = {0}; - bool isDrop; - - // I am in newConfig - if (hit) { - syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop); - - // change isStandBy to normal - if (!isDrop) { - if (ths->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(ths); - } else { - syncNodeBecomeFollower(ths); - } - } - - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); - } - - // always call FpReConfigCb - if (ths->pFsm->FpReConfigCb != NULL) { - cbMeta.code = 0; - cbMeta.currentTerm = ths->pRaftStore->currentTerm; - cbMeta.index = pEntry->index; - cbMeta.term = pEntry->term; - cbMeta.oldCfg = oldSyncCfg; - cbMeta.flag = 0x11; - cbMeta.isDrop = isDrop; - ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta); - } + // config change + if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { + code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); + ASSERT(code == 0); } // restore finish @@ -1824,7 +1933,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sInfo("restore finish %p vgId:%d", ths, ths->vgId); + sDebug("vgId:%d sync event restore finish", ths->vgId); } } @@ -1853,4 +1962,4 @@ SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) } } return pSender; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index af04a0f649..23165f6790 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -14,6 +14,7 @@ */ #include "syncMessage.h" +#include "syncRaftCfg.h" #include "syncUtil.h" #include "tcoding.h" @@ -75,6 +76,11 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { pRoot = syncSnapshotRsp2Json(pSyncMsg); syncSnapshotRspDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { + SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncLeaderTransfer2Json(pSyncMsg); + syncLeaderTransferDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { pRoot = cJSON_CreateObject(); char* s; @@ -1841,6 +1847,10 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastConfigIndex); + cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); + cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); @@ -2055,4 +2065,166 @@ void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg) { sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } +} + +// --------------------------------------------- +SyncLeaderTransfer* syncLeaderTransferBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncLeaderTransfer); + SyncLeaderTransfer* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_LEADER_TRANSFER; + return pMsg; +} + +void syncLeaderTransferDestroy(SyncLeaderTransfer* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncLeaderTransferSerialize(const SyncLeaderTransfer* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncLeaderTransferDeserialize(const char* buf, uint32_t len, SyncLeaderTransfer* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +char* syncLeaderTransferSerialize2(const SyncLeaderTransfer* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + assert(buf != NULL); + syncLeaderTransferSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncLeaderTransfer* syncLeaderTransferDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncLeaderTransfer* pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + syncLeaderTransferDeserialize(buf, len, pMsg); + assert(len == pMsg->bytes); + return pMsg; +} + +void syncLeaderTransfer2RpcMsg(const SyncLeaderTransfer* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncLeaderTransferSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncLeaderTransferFromRpcMsg(const SRpcMsg* pRpcMsg, SyncLeaderTransfer* pMsg) { + syncLeaderTransferDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncLeaderTransfer* syncLeaderTransferFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncLeaderTransfer* pMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + assert(pMsg != NULL); + return pMsg; +} + +cJSON* syncLeaderTransfer2Json(const SyncLeaderTransfer* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + /* + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + */ + + cJSON* pNewerId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->newLeaderId.addr); + cJSON_AddStringToObject(pNewerId, "addr", u64buf); + { + uint64_t u64 = pMsg->newLeaderId.addr; + cJSON* pTmp = pNewerId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pNewerId, "vgId", pMsg->newLeaderId.vgId); + cJSON_AddItemToObject(pRoot, "newLeaderId", pNewerId); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncLeaderTransfer", pRoot); + return pJson; +} + +char* syncLeaderTransfer2Str(const SyncLeaderTransfer* pMsg) { + cJSON* pJson = syncLeaderTransfer2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncLeaderTransferPrint(const SyncLeaderTransfer* pMsg) { + char* serialized = syncLeaderTransfer2Str(pMsg); + printf("syncLeaderTransferPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncLeaderTransferPrint2(char* s, const SyncLeaderTransfer* pMsg) { + char* serialized = syncLeaderTransfer2Str(pMsg); + printf("syncLeaderTransferPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncLeaderTransferLog(const SyncLeaderTransfer* pMsg) { + char* serialized = syncLeaderTransfer2Str(pMsg); + sTrace("syncLeaderTransferLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncLeaderTransferLog2(char* s, const SyncLeaderTransfer* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncLeaderTransfer2Str(pMsg); + sTrace("syncLeaderTransferLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 95eec5d98f..45e00aca2c 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -150,6 +150,10 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy); cJSON_AddNumberToObject(pRoot, "snapshotEnable", pRaftCfg->snapshotEnable); + char buf64[128]; + snprintf(buf64, sizeof(buf64), "%ld", pRaftCfg->lastConfigIndex); + cJSON_AddStringToObject(pRoot, "lastConfigIndex", buf64); + cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RaftCfg", pRoot); return pJson; @@ -172,6 +176,7 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { raftCfg.cfg = *pCfg; raftCfg.isStandBy = meta.isStandBy; raftCfg.snapshotEnable = meta.snapshotEnable; + raftCfg.lastConfigIndex = meta.lastConfigIndex; char *s = raftCfg2Str(&raftCfg); char buf[CONFIG_FILE_LEN] = {0}; @@ -199,6 +204,9 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { cJSON *pJsonSnapshotEnable = cJSON_GetObjectItem(pJson, "snapshotEnable"); pRaftCfg->snapshotEnable = cJSON_GetNumberValue(pJsonSnapshotEnable); + cJSON *pJsonLastConfigIndex = cJSON_GetObjectItem(pJson, "lastConfigIndex"); + pRaftCfg->lastConfigIndex = atoll(cJSON_GetStringValue(pJsonLastConfigIndex)); + cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 49509ae979..996cd12e4a 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -14,6 +14,7 @@ */ #include "syncRaftLog.h" +#include "syncRaftCfg.h" #include "wal.h" // refactor, log[0 .. n] ==> log[m .. n] @@ -161,7 +162,9 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); - sTrace("sync event write index:%" PRId64, pEntry->index); + sDebug("vgId:%d sync event write index:%ld, %s, isStandBy:%d, msgType:%s, originalRpcType:%s", pData->pSyncNode->vgId, + pEntry->index, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftCfg->isStandBy, + TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType)); return code; } @@ -317,7 +320,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { walFsync(pWal, true); - sTrace("sync event old write wal: %ld", pEntry->index); + sDebug("sync event old write wal: %ld", pEntry->index); return code; } @@ -550,15 +553,19 @@ void logStorePrint2(char* s, SSyncLogStore* pLogStore) { } void logStoreLog(SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - sTraceLong("logStoreLog | len:%lu | %s", strlen(serialized), serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sTraceLong("logStoreLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); + } } void logStoreLog2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - sTraceLong("logStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sTraceLong("logStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // for debug ----------------- diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index ff39b0b13d..08564f8293 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -122,7 +122,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { syncIndexMgrLog2("begin append entries peers pNextIndex:", pSyncNode->pNextIndex); syncIndexMgrLog2("begin append entries peers pMatchIndex:", pSyncNode->pMatchIndex); logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore); - { + if (gRaftDetailLog) { SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); sTrace("begin append entries peers, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", @@ -201,7 +201,6 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) { } int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) { - sTrace("syncNodeAppendEntries pSyncNode:%p ", pSyncNode); int32_t ret = 0; SRpcMsg rpcMsg; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a68312d07f..38742220fe 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -15,12 +15,13 @@ #include "syncSnapshot.h" #include "syncIndexMgr.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "wal.h" -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm); +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SRaftId fromId); //---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { @@ -83,6 +84,32 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { // get current snapshot info pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { + /* + SSyncRaftEntry *pEntry = NULL; + int32_t code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, + pSender->snapshot.lastConfigIndex, &pEntry); + ASSERT(code == 0); + ASSERT(pEntry != NULL); + */ + + SSyncRaftEntry *pEntry = + pSender->pSyncNode->pLogStore->getEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex); + ASSERT(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + SSyncCfg lastConfig; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &lastConfig); + ASSERT(ret == 0); + pSender->lastConfig = lastConfig; + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + + } else { + memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); + } pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; @@ -97,6 +124,8 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; // SYNC_SNAPSHOT_SEQ_BEGIN pMsg->privateTerm = pSender->privateTerm; @@ -105,13 +134,26 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - char *msgStr = syncSnapshotSend2Str(pMsg); char host[128]; uint16_t port; syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); - sTrace("sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", host, - port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, msgStr); - taosMemoryFree(msgStr); + + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug( + "vgId:%d sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "lastConfigIndex:%ld send " + "msg:%s", + pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, msgStr); + taosMemoryFree(msgStr); + } else { + sDebug( + "vgId:%d sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "lastConfigIndex:%ld", + pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); + } syncSnapshotSendDestroy(pMsg); } @@ -183,9 +225,11 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender) { pSender->start = false; - char *s = snapshotSender2Str(pSender); - sInfo("snapshotSenderStop %s", s); - taosMemoryFree(s); + if (gRaftDetailLog) { + char *s = snapshotSender2Str(pSender); + sInfo("snapshotSenderStop %s", s); + taosMemoryFree(s); + } } // when sender receiver ack, call this function to send msg from seq @@ -216,6 +260,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); @@ -225,20 +271,34 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - char *msgStr = syncSnapshotSend2Str(pMsg); char host[128]; uint16_t port; syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - sTrace("sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", - host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - msgStr); + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug( + "vgId:%d sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "lastConfigIndex:%ld send " + "msg:%s", + pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, msgStr); + taosMemoryFree(msgStr); + } else { + sDebug( + "vgId:%d sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "lastConfigIndex:%ld", + pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); + } } else { - sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", - host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - msgStr); + sDebug( + "vgId:%d sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "lastConfigIndex:%ld", + pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); } - taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); return 0; @@ -253,6 +313,8 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); @@ -260,13 +322,19 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - char *msgStr = syncSnapshotSend2Str(pMsg); char host[128]; uint16_t port; syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); - sTrace("sync event snapshot send to %s:%d resend seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, - msgStr); - taosMemoryFree(msgStr); + + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d send msg:%s", pSender->pSyncNode->vgId, + host, port, pSender->seq, pSender->ack, msgStr); + taosMemoryFree(msgStr); + } else { + sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d", pSender->pSyncNode->vgId, host, port, + pSender->seq, pSender->ack); + } syncSnapshotSendDestroy(pMsg); } @@ -325,13 +393,13 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } // ------------------------------------- -SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { +SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); @@ -345,7 +413,7 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; pReceiver->pWriter = NULL; pReceiver->pSyncNode = pSyncNode; - pReceiver->replicaIndex = replicaIndex; + pReceiver->fromId = fromId; pReceiver->term = pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = 0; @@ -365,10 +433,11 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } // begin receive snapshot msg (current term, seq begin) -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) { +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SRaftId fromId) { pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = privateTerm; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + pReceiver->fromId = fromId; ASSERT(pReceiver->pWriter == NULL); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); @@ -377,14 +446,15 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm p // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver // if already start, force close, start again -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) { +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SRaftId fromId) { if (!snapshotReceiverIsStart(pReceiver)) { // start - snapshotReceiverDoStart(pReceiver, privateTerm); + snapshotReceiverDoStart(pReceiver, privateTerm, fromId); pReceiver->start = true; } else { // already start + sInfo("snapshot recv, receiver already start"); // force close, abandon incomplete data int32_t ret = @@ -393,15 +463,15 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTer pReceiver->pWriter = NULL; // start again - snapshotReceiverDoStart(pReceiver, privateTerm); + snapshotReceiverDoStart(pReceiver, privateTerm, fromId); pReceiver->start = true; - - ASSERT(0); } - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStart %s", s); - taosMemoryFree(s); + if (gRaftDetailLog) { + char *s = snapshotReceiver2Str(pReceiver); + sInfo("snapshotReceiverStart %s", s); + taosMemoryFree(s); + } } void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { @@ -418,9 +488,11 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { ++(pReceiver->privateTerm); } - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStop %s", s); - taosMemoryFree(s); + if (gRaftDetailLog) { + char *s = snapshotReceiver2Str(pReceiver); + sInfo("snapshotReceiverStop %s", s); + taosMemoryFree(s); + } } cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { @@ -436,7 +508,22 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "replicaIndex", pReceiver->replicaIndex); + + cJSON *pFromId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->fromId.addr); + cJSON_AddStringToObject(pFromId, "addr", u64buf); + { + uint64_t u64 = pReceiver->fromId.addr; + cJSON * pTmp = pFromId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); + cJSON_AddItemToObject(pRoot, "fromId", pFromId); + snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->term); cJSON_AddStringToObject(pRoot, "term", u64buf); @@ -451,7 +538,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -468,17 +555,28 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { // begin - snapshotReceiverStart(pReceiver, pMsg->privateTerm); + snapshotReceiverStart(pReceiver, pMsg->privateTerm, pMsg->srcId); pReceiver->ack = pMsg->seq; needRsp = true; - char *msgStr = syncSnapshotSend2Str(pMsg); char host[128]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sTrace("sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port, - pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); - taosMemoryFree(msgStr); + + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug( + "vgId:%d sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld, recv msg:%s", + pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, + msgStr); + taosMemoryFree(msgStr); + } else { + sDebug( + "vgId:%d sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld", + pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { // end, finish FSM @@ -486,29 +584,89 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { ASSERT(writeCode == 0); pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); - pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); - char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); + + // maybe update lastconfig + if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { + // update new config myIndex + bool IamInNew = false; + SSyncCfg newSyncCfg = pMsg->lastConfig; + for (int i = 0; i < newSyncCfg.replicaNum; ++i) { + if (strcmp(pSyncNode->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && + pSyncNode->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { + newSyncCfg.myIndex = i; + IamInNew = true; + break; + } + } + + bool isDrop; + if (IamInNew) { + sDebug("vgId:%d sync event update config by snapshot, lastIndex:%ld, lastTerm:%lu, lastConfigIndex:%ld ", + pSyncNode->vgId, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + syncNodeUpdateConfig(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex, &isDrop); + } else { + sDebug( + "vgId:%d sync event do not update config by snapshot, I am not in newCfg, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld ", + pSyncNode->vgId, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + } + + // change isStandBy to normal + if (!isDrop) { + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(pSyncNode, "config change"); + } else { + syncNodeBecomeFollower(pSyncNode, "config change"); + } + } + } + SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + char host[128]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sInfo( - "sync event snapshot recv from %s:%d finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " - "snapshot.lastApplyTerm:%lu, raft log:%s", - host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); - taosMemoryFree(logSimpleStr); + + if (gRaftDetailLog) { + char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); + sDebug( + "vgId:%d sync event snapshot recv from %s:%d finish, update log begin index:%ld, " + "snapshot.lastApplyIndex:%ld, " + "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, raft log:%s", + pSyncNode->vgId, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, + snapshot.lastConfigIndex, logSimpleStr); + taosMemoryFree(logSimpleStr); + } else { + sDebug( + "vgId:%d sync event snapshot recv from %s:%d finish, update log begin index:%ld, " + "snapshot.lastApplyIndex:%ld, " + "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld", + pSyncNode->vgId, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, + snapshot.lastConfigIndex); + } pReceiver->pWriter = NULL; snapshotReceiverStop(pReceiver, true); pReceiver->ack = pMsg->seq; needRsp = true; - char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port, - pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); - taosMemoryFree(msgStr); + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug( + "vgId:%d sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld, recv msg:%s", + pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex, msgStr); + taosMemoryFree(msgStr); + } else { + sDebug( + "vgId:%d sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld", + pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex); + } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); @@ -519,11 +677,22 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, - port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); - - taosMemoryFree(msgStr); + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug( + "vgId:%d sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld, recv " + "msg:%s", + pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex, msgStr); + taosMemoryFree(msgStr); + } else { + sDebug( + "vgId:%d sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld", + pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex); + } } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // transfering @@ -535,13 +704,24 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } needRsp = true; - char *msgStr = syncSnapshotSend2Str(pMsg); char host[128]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sTrace("sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, - port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); - taosMemoryFree(msgStr); + + if (gRaftDetailLog) { + char *msgStr = syncSnapshotSend2Str(pMsg); + sDebug( + "vgId:%d sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld, recv msg:%s", + pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, + msgStr); + taosMemoryFree(msgStr); + } else { + sDebug( + "vgId:%d sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld", + pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + } } else { ASSERT(0); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index f6ff521e01..d12c5058cc 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -214,29 +214,31 @@ void syncUtilMsgNtoH(void* msg) { pHead->vgId = ntohl(pHead->vgId); } +#if 0 bool syncUtilIsData(tmsg_t msgType) { if (msgType == TDMT_SYNC_NOOP || msgType == TDMT_SYNC_CONFIG_CHANGE) { return false; } return true; } +#endif bool syncUtilUserPreCommit(tmsg_t msgType) { - if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE) { + if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE && msgType != TDMT_SYNC_LEADER_TRANSFER) { return true; } return false; } bool syncUtilUserCommit(tmsg_t msgType) { - if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE) { + if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE && msgType != TDMT_SYNC_LEADER_TRANSFER) { return true; } return false; } bool syncUtilUserRollback(tmsg_t msgType) { - if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE) { + if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE && msgType != TDMT_SYNC_LEADER_TRANSFER) { return true; } return false; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index c68c6349fb..d39035ba53 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -47,6 +47,7 @@ add_executable(syncTestTool "") add_executable(syncRaftLogTest "") add_executable(syncRaftLogTest2 "") add_executable(syncRaftLogTest3 "") +add_executable(syncLeaderTransferTest "") target_sources(syncTest @@ -245,6 +246,10 @@ target_sources(syncRaftLogTest3 PRIVATE "syncRaftLogTest3.cpp" ) +target_sources(syncLeaderTransferTest + PRIVATE + "syncLeaderTransferTest.cpp" +) target_include_directories(syncTest @@ -492,6 +497,11 @@ target_include_directories(syncRaftLogTest3 "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncLeaderTransferTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -690,6 +700,10 @@ target_link_libraries(syncRaftLogTest3 sync gtest_main ) +target_link_libraries(syncLeaderTransferTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 781c168da9..10b54d0aa4 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -146,7 +146,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } -void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { +void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta) { sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu", cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term); } diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index c9d9ca48aa..1e64a8a6f7 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -77,7 +77,7 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } -void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { +void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta) { sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu", cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term); } diff --git a/source/libs/sync/test/syncLeaderTransferTest.cpp b/source/libs/sync/test/syncLeaderTransferTest.cpp new file mode 100644 index 0000000000..1c3891d492 --- /dev/null +++ b/source/libs/sync/test/syncLeaderTransferTest.cpp @@ -0,0 +1,101 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncLeaderTransfer *createMsg() { + SyncLeaderTransfer *pMsg = syncLeaderTransferBuild(1000); + /* + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + */ + pMsg->newLeaderId.addr = syncUtilAddr2U64("127.0.0.1", 9999); + pMsg->newLeaderId.vgId = 100; + return pMsg; +} + +void test1() { + SyncLeaderTransfer *pMsg = createMsg(); + syncLeaderTransferLog2((char *)"test1:", pMsg); + syncLeaderTransferDestroy(pMsg); +} + +void test2() { + SyncLeaderTransfer *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncLeaderTransferSerialize(pMsg, serialized, len); + SyncLeaderTransfer *pMsg2 = syncLeaderTransferBuild(1000); + syncLeaderTransferDeserialize(serialized, len, pMsg2); + syncLeaderTransferLog2((char *)"test2: syncLeaderTransferSerialize -> syncLeaderTransferDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncLeaderTransferDestroy(pMsg); + syncLeaderTransferDestroy(pMsg2); +} + +void test3() { + SyncLeaderTransfer *pMsg = createMsg(); + uint32_t len; + char * serialized = syncLeaderTransferSerialize2(pMsg, &len); + SyncLeaderTransfer *pMsg2 = syncLeaderTransferDeserialize2(serialized, len); + syncLeaderTransferLog2((char *)"test3: syncLeaderTransferSerialize2 -> syncLeaderTransferDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncLeaderTransferDestroy(pMsg); + syncLeaderTransferDestroy(pMsg2); +} + +void test4() { + SyncLeaderTransfer *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); + SyncLeaderTransfer *pMsg2 = (SyncLeaderTransfer *)taosMemoryMalloc(rpcMsg.contLen); + syncLeaderTransferFromRpcMsg(&rpcMsg, pMsg2); + syncLeaderTransferLog2((char *)"test4: syncLeaderTransfer2RpcMsg -> syncLeaderTransferFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncLeaderTransferDestroy(pMsg); + syncLeaderTransferDestroy(pMsg2); +} + +void test5() { + SyncLeaderTransfer *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); + SyncLeaderTransfer *pMsg2 = syncLeaderTransferFromRpcMsg2(&rpcMsg); + syncLeaderTransferLog2((char *)"test5: syncLeaderTransfer2RpcMsg -> syncLeaderTransferFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncLeaderTransferDestroy(pMsg); + syncLeaderTransferDestroy(pMsg2); +} + +int main() { + gRaftDetailLog = true; + + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index 564cbdb69a..8c6a704e2d 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -74,6 +74,7 @@ void test3() { SRaftCfgMeta meta; meta.isStandBy = 7; meta.snapshotEnable = 9; + meta.lastConfigIndex = 789; raftCfgCreateFile(pCfg, meta, s); printf("%s create json file: %s \n", (char*)__FUNCTION__, s); } @@ -98,6 +99,7 @@ void test5() { pCfg->cfg.myIndex = taosGetTimestampSec(); pCfg->isStandBy += 2; pCfg->snapshotEnable += 3; + pCfg->lastConfigIndex += 1000; raftCfgPersist(pCfg); printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 69670f09a6..208a96daa4 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -41,7 +41,11 @@ SSyncSnapshotReceiver* createReceiver() { pSyncNode->pFsm->FpSnapshotStopWrite = SnapshotStopWrite; pSyncNode->pFsm->FpSnapshotDoWrite = SnapshotDoWrite; - SSyncSnapshotReceiver* pReceiver = snapshotReceiverCreate(pSyncNode, 2); + SRaftId id; + id.addr = syncUtilAddr2U64("1.2.3.4", 99); + id.vgId = 100; + + SSyncSnapshotReceiver* pReceiver = snapshotReceiverCreate(pSyncNode, id); pReceiver->start = true; pReceiver->ack = 20; pReceiver->pWriter = (void*)0x11; diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 01d3264693..ca7916359e 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -24,6 +24,16 @@ SyncSnapshotSend *createMsg() { pMsg->privateTerm = 99; pMsg->lastIndex = 22; pMsg->lastTerm = 33; + + pMsg->lastConfigIndex = 99; + pMsg->lastConfig.replicaNum = 3; + pMsg->lastConfig.myIndex = 1; + for (int i = 0; i < pMsg->lastConfig.replicaNum; ++i) { + ((pMsg->lastConfig.nodeInfo)[i]).nodePort = i * 100; + snprintf(((pMsg->lastConfig.nodeInfo)[i]).nodeFqdn, sizeof(((pMsg->lastConfig.nodeInfo)[i]).nodeFqdn), + "100.200.300.%d", i); + } + pMsg->seq = 44; strcpy(pMsg->data, "hello world"); return pMsg; @@ -87,6 +97,8 @@ void test5() { } int main() { + gRaftDetailLog = true; + tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 782baf3c97..0c8b26e9d9 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -146,8 +146,8 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb== pFsm:%p", pFsm); } -void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { - char* s = syncCfg2Str(&newCfg); +void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta) { + char* s = syncCfg2Str(&(cbMeta.newCfg)); sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu, newCfg:%s", cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term, s); taosMemoryFree(s); @@ -235,7 +235,6 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* } } - int64_t rid = syncOpen(&syncInfo); assert(rid > 0); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 3b68073c7e..11f62455fd 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -81,24 +81,24 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { rel.tv_nsec = nanosecs; GetSystemTimeAsFileTime(&ft_before); - errno = 0; - rc = sem_timedwait(&sem, pthread_win32_getabstime_np(&ts, &rel)); + // errno = 0; + rc = sem_timedwait(sem, pthread_win32_getabstime_np(&ts, &rel)); /* This should have timed out */ - assert(errno == ETIMEDOUT); - assert(rc != 0); - GetSystemTimeAsFileTime(&ft_after); - // We specified a non-zero wait. Time must advance. - if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) - { - printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n", - nanosecs, rc, errno, - (int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime, - (int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime); - printf("time must advance during sem_timedwait."); - return 1; - } - return 0; + // assert(errno == ETIMEDOUT); + // assert(rc != 0); + // GetSystemTimeAsFileTime(&ft_after); + // // We specified a non-zero wait. Time must advance. + // if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) + // { + // printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n", + // nanosecs, rc, errno, + // (int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime, + // (int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime); + // printf("time must advance during sem_timedwait."); + // return 1; + // } + return rc; } #elif defined(_TD_DARWIN_64) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b5c54b780d..e122ad0ab6 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -353,9 +353,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag valu TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "TSMA already exists") // query @@ -537,25 +534,38 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statemen TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes") //planner -TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") //udf -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") //schemaless -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") + +//tsma +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_ALREADY_EXIST, "Tsma already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_META, "No tsma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_ENV, "Invalid tsma env") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_STAT, "Invalid tsma state") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in cache") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_RM_SKEY_IN_HASH, "Rm tsma skey in cache") + +//rsma +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") + #ifdef TAOS_ERROR_C }; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index b15c188f04..45a2ffec77 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "tjson.h" #include "cJSON.h" #include "taoserror.h" @@ -138,6 +139,23 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* return TSDB_CODE_SUCCESS; } +int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArray* pArray) { + int32_t num = taosArrayGetSize(pArray); + if (num > 0) { + SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName); + if (NULL == pJsonArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < num; ++i) { + int32_t code = tjsonAddItem(pJsonArray, func, taosArrayGet(pArray, i)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } @@ -184,7 +202,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p,"%lld",pVal); + sscanf(p, "%lld", pVal); #else // sscanf(p,"%ld",pVal); *pVal = taosStr2Int64(p, NULL, 10); @@ -219,7 +237,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p,"%llu",pVal); + sscanf(p, "%llu", pVal); #else // sscanf(p,"%ld",pVal); *pVal = taosStr2UInt64(p, NULL, 10); @@ -299,24 +317,43 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void return TSDB_CODE_SUCCESS; } +int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SArray** pArray, int32_t itemSize) { + const cJSON* jArray = tjsonGetObjectItem(pJson, pName); + int32_t size = tjsonGetArraySize(jArray); + if (size > 0) { + *pArray = taosArrayInit(size, itemSize); + if (NULL == *pArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArraySetSize(*pArray, size); + for (int32_t i = 0; i < size; ++i) { + int32_t code = func(tjsonGetArrayItem(jArray, i), taosArrayGet(*pArray, i)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } -bool tjsonValidateJson(const char *jIn) { - if (!jIn){ +bool tjsonValidateJson(const char* jIn) { + if (!jIn) { return false; } // set json real data - cJSON *root = cJSON_Parse(jIn); - if (root == NULL){ + cJSON* root = cJSON_Parse(jIn); + if (root == NULL) { return false; } - if(!cJSON_IsObject(root)){ + if (!cJSON_IsObject(root)) { return false; } int size = cJSON_GetArraySize(root); - for(int i = 0; i < size; i++) { + for (int i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { return false; diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c new file mode 100644 index 0000000000..b034a6e73e --- /dev/null +++ b/source/util/src/tlrucache.c @@ -0,0 +1,794 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tlrucache.h" +#include "os.h" +#include "tdef.h" +#include "taoserror.h" +#include "tlog.h" +#include "tarray.h" + +typedef struct SLRUEntry SLRUEntry; +typedef struct SLRUEntryTable SLRUEntryTable; +typedef struct SLRUCacheShard SLRUCacheShard; +typedef struct SShardedCache SShardedCache; + +enum { + TAOS_LRU_IN_CACHE = (1 << 0), // Whether this entry is referenced by the hash table. + + TAOS_LRU_IS_HIGH_PRI = (1 << 1), // Whether this entry is high priority entry. + + TAOS_LRU_IN_HIGH_PRI_POOL = (1 << 2), // Whether this entry is in high-pri pool. + + TAOS_LRU_HAS_HIT = (1 << 3), // Whether this entry has had any lookups (hits). +}; + +struct SLRUEntry { + void *value; + _taos_lru_deleter_t deleter; + SLRUEntry *nextHash; + SLRUEntry *next; + SLRUEntry *prev; + size_t totalCharge; + size_t keyLength; + uint32_t hash; + uint32_t refs; + uint8_t flags; + char keyData[1]; +}; + +#define TAOS_LRU_ENTRY_IN_CACHE(h) ((h)->flags & TAOS_LRU_IN_CACHE) +#define TAOS_LRU_ENTRY_IN_HIGH_POOL(h) ((h)->flags & TAOS_LRU_IN_HIGH_PRI_POOL) +#define TAOS_LRU_ENTRY_IS_HIGH_PRI(h) ((h)->flags & TAOS_LRU_IS_HIGH_PRI) +#define TAOS_LRU_ENTRY_HAS_HIT(h) ((h)->flags & TAOS_LRU_HAS_HIT) + +#define TAOS_LRU_ENTRY_SET_IN_CACHE(h, inCache) do { if(inCache) {(h)->flags |= TAOS_LRU_IN_CACHE;} else {(h)->flags &= ~TAOS_LRU_IN_CACHE;} } while(0) +#define TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(h, inHigh) do { if(inHigh) {(h)->flags |= TAOS_LRU_IN_HIGH_PRI_POOL;} else {(h)->flags &= ~TAOS_LRU_IN_HIGH_PRI_POOL;} } while(0) +#define TAOS_LRU_ENTRY_SET_PRIORITY(h, priority) do { if(priority == TAOS_LRU_PRIORITY_HIGH) {(h)->flags |= TAOS_LRU_IS_HIGH_PRI;} else {(h)->flags &= ~TAOS_LRU_IS_HIGH_PRI;} } while(0) +#define TAOS_LRU_ENTRY_SET_HIT(h) ((h)->flags |= TAOS_LRU_HAS_HIT) + +#define TAOS_LRU_ENTRY_HAS_REFS(h) ((h)->refs > 0) +#define TAOS_LRU_ENTRY_REF(h) (++(h)->refs) + +static bool taosLRUEntryUnref(SLRUEntry *entry) { + assert(entry->refs > 0); + --entry->refs; + return entry->refs == 0; +} + +static void taosLRUEntryFree(SLRUEntry *entry) { + assert(entry->refs == 0); + + if (entry->deleter) { + (*entry->deleter)(entry->keyData, entry->keyLength, entry->value); + } + + taosMemoryFree(entry); +} + +typedef void (*_taos_lru_table_func_t)(SLRUEntry *entry); + +struct SLRUEntryTable { + int lengthBits; + SLRUEntry **list; + uint32_t elems; + int maxLengthBits; +}; + +static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { + table->lengthBits = 4; + table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry*)); + if (!table->list) { + return -1; + } + + table->elems = 0; + table->maxLengthBits = maxUpperHashBits; + + return 0; +} + +static void taosLRUEntryTableApply(SLRUEntryTable *table, _taos_lru_table_func_t func, uint32_t begin, uint32_t end) { + for (uint32_t i = begin; i < end; ++i) { + SLRUEntry *h = table->list[i]; + while (h) { + SLRUEntry *n = h->nextHash; + assert(TAOS_LRU_ENTRY_IN_CACHE(h)); + func(h); + h = n; + } + } +} + +static void taosLRUEntryTableFree(SLRUEntry *entry) { + if (!TAOS_LRU_ENTRY_HAS_REFS(entry)) { + taosLRUEntryFree(entry); + } +} + +static void taosLRUEntryTableCleanup(SLRUEntryTable *table) { + taosLRUEntryTableApply(table, taosLRUEntryTableFree, 0, 1 << table->lengthBits); + + taosMemoryFree(table->list); +} + +static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) { + SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)]; + while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) { + entry = &(*entry)->nextHash; + } + + return entry; +} + +static void taosLRUEntryTableResize(SLRUEntryTable * table) { + int lengthBits = table->lengthBits; + if (lengthBits >= table->maxLengthBits) { + return; + } + + if (lengthBits >= 31) { + return; + } + + uint32_t oldLength = 1 << lengthBits; + int newLengthBits = lengthBits + 1; + SLRUEntry **newList = taosMemoryCalloc(1 << newLengthBits, sizeof(SLRUEntry*)); + if (!newList) { + return; + } + uint32_t count = 0; + for (uint32_t i = 0; i < oldLength; ++i) { + SLRUEntry *entry = table->list[i]; + while (entry) { + SLRUEntry *next = entry->nextHash; + uint32_t hash = entry->hash; + SLRUEntry **ptr = &newList[hash >> (32 - newLengthBits)]; + entry->nextHash = *ptr; + *ptr = entry; + entry = next; + ++count; + } + } + assert(table->elems == count); + + taosMemoryFree(table->list); + table->list = newList; + table->lengthBits = newLengthBits; +} + +static SLRUEntry *taosLRUEntryTableLookup(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) { + return *taosLRUEntryTableFindPtr(table, key, keyLen, hash); +} + +static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable * table, SLRUEntry *entry) { + SLRUEntry **ptr = taosLRUEntryTableFindPtr(table, entry->keyData, entry->keyLength, entry->hash); + SLRUEntry *old = *ptr; + entry->nextHash = (old == NULL) ? NULL : old->nextHash; + *ptr = entry; + if (old == NULL) { + ++table->elems; + if ((table->elems >> table->lengthBits) > 0) { + taosLRUEntryTableResize(table); + } + } + + return old; +} + +static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) { + SLRUEntry **entry = taosLRUEntryTableFindPtr(table, key, keyLen, hash); + SLRUEntry *result = *entry; + if (result) { + *entry = result->nextHash; + --table->elems; + } + + return result; +} + +struct SLRUCacheShard { + size_t capacity; + size_t highPriPoolUsage; + bool strictCapacity; + double highPriPoolRatio; + double highPriPoolCapacity; + SLRUEntry lru; + SLRUEntry *lruLowPri; + SLRUEntryTable table; + size_t usage; // Memory size for entries residing in the cache. + size_t lruUsage; // Memory size for entries residing only in the LRU list. + TdThreadMutex mutex; +}; + +#define TAOS_LRU_CACHE_SHARD_HASH32(key, len) (MurmurHash3_32((key), (len))) + +static void taosLRUCacheShardMaintainPoolSize(SLRUCacheShard *shard) { + while (shard->highPriPoolUsage > shard->highPriPoolCapacity) { + shard->lruLowPri = shard->lruLowPri->next; + assert(shard->lruLowPri != &shard->lru); + TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(shard->lruLowPri, false); + + assert(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge); + shard->highPriPoolUsage -= shard->lruLowPri->totalCharge; + } +} + +static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { + assert(e->next == NULL); + assert(e->prev == NULL); + + if (shard->highPriPoolRatio > 0 + && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) { + e->next = &shard->lru; + e->prev = shard->lru.prev; + + e->prev->next = e; + e->next->prev = e; + + TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(e, true); + shard->highPriPoolUsage += e->totalCharge; + taosLRUCacheShardMaintainPoolSize(shard); + } else { + e->next = shard->lruLowPri->next; + e->prev = shard->lruLowPri; + + e->prev->next = e; + e->next->prev = e; + + TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(e, false); + shard->lruLowPri = e; + } + + shard->lruUsage += e->totalCharge; +} + +static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { + assert(e->next); + assert(e->prev); + + if (shard->lruLowPri == e) { + shard->lruLowPri = e->prev; + } + e->next->prev = e->prev; + e->prev->next = e->next; + e->prev = e->next = NULL; + + assert(shard->lruUsage >= e->totalCharge); + shard->lruUsage -= e->totalCharge; + if (TAOS_LRU_ENTRY_IN_HIGH_POOL(e)) { + assert(shard->highPriPoolUsage >= e->totalCharge); + shard->highPriPoolUsage -= e->totalCharge; + } +} + +static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArray *deleted) { + while (shard->usage + charge > shard->capacity && shard->lru.next != &shard->lru) { + SLRUEntry *old = shard->lru.next; + assert(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); + + taosLRUCacheShardLRURemove(shard, old); + taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); + + TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); + assert(shard->usage >= old->totalCharge); + shard->usage -= old->totalCharge; + + taosArrayPush(deleted, &old); + } +} + +static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity) { + SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); + + taosThreadMutexLock(&shard->mutex); + + shard->capacity = capacity; + shard->highPriPoolCapacity = capacity * shard->highPriPoolRatio; + taosLRUCacheShardEvictLRU(shard, 0, lastReferenceList); + + taosThreadMutexUnlock(&shard->mutex); + + for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { + SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); + taosLRUEntryFree(entry); + } + taosArrayDestroy(lastReferenceList); +} + +static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool strict, + double highPriPoolRatio, int maxUpperHashBits) { + if (taosLRUEntryTableInit(&shard->table, maxUpperHashBits) < 0) { + return -1; + } + + taosThreadMutexInit(&shard->mutex, NULL); + + shard->capacity = 0; + shard->highPriPoolUsage = 0; + shard->strictCapacity = strict; + shard->highPriPoolRatio = highPriPoolRatio; + shard->highPriPoolCapacity = 0; + + shard->usage = 0; + shard->lruUsage = 0; + + shard->lru.next = &shard->lru; + shard->lru.prev = &shard->lru; + shard->lruLowPri = &shard->lru; + + taosLRUCacheShardSetCapacity(shard, capacity); + + return 0; +} + +static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { + taosThreadMutexDestroy(&shard->mutex); + + taosLRUEntryTableCleanup(&shard->table); +} + +static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { + LRUStatus status = TAOS_LRU_STATUS_OK; + SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); + + taosThreadMutexLock(&shard->mutex); + + taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList); + + if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) { + TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); + if (handle == NULL) { + taosArrayPush(lastReferenceList, &e); + } else { + if (freeOnFail) { + taosMemoryFree(e); + + *handle = NULL; + } + + status = TAOS_LRU_STATUS_INCOMPLETE; + } + } else { + SLRUEntry *old = taosLRUEntryTableInsert(&shard->table, e); + shard->usage += e->totalCharge; + if (old != NULL) { + status = TAOS_LRU_STATUS_OK_OVERWRITTEN; + + assert(TAOS_LRU_ENTRY_IN_CACHE(old)); + TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); + if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { + taosLRUCacheShardLRURemove(shard, old); + assert(shard->usage >= old->totalCharge); + shard->usage -= old->totalCharge; + + taosArrayPush(lastReferenceList, &old); + } + } + if (handle == NULL) { + taosLRUCacheShardLRUInsert(shard, e); + } else { + if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { + TAOS_LRU_ENTRY_REF(e); + } + + *handle = (LRUHandle*) e; + } + } + + taosThreadMutexUnlock(&shard->mutex); + + for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { + SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); + + taosLRUEntryFree(entry); + } + taosArrayDestroy(lastReferenceList); + + return status; +} + +static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash, + void *value, size_t charge, _taos_lru_deleter_t deleter, + LRUHandle **handle, LRUPriority priority) { + SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen); + if (!e) { + return TAOS_LRU_STATUS_FAIL; + } + + e->value = value; + e->flags = 0; + e->deleter = deleter; + e->keyLength = keyLen; + e->hash = hash; + e->refs = 0; + e->next = e->prev = NULL; + TAOS_LRU_ENTRY_SET_IN_CACHE(e, true); + + TAOS_LRU_ENTRY_SET_PRIORITY(e, priority); + memcpy(e->keyData, key, keyLen); + // TODO: e->CalcTotalCharge(charge, metadataChargePolicy); + e->totalCharge = charge; + + return taosLRUCacheShardInsertEntry(shard, e, handle, true); +} + +static LRUHandle *taosLRUCacheShardLookup(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash) { + SLRUEntry *e = NULL; + + taosThreadMutexLock(&shard->mutex); + e = taosLRUEntryTableLookup(&shard->table, key, keyLen, hash); + if (e != NULL) { + assert(TAOS_LRU_ENTRY_IN_CACHE(e)); + if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { + taosLRUCacheShardLRURemove(shard, e); + } + TAOS_LRU_ENTRY_REF(e); + TAOS_LRU_ENTRY_SET_HIT(e); + } + + taosThreadMutexUnlock(&shard->mutex); + + return (LRUHandle *) e; +} + +static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash) { + bool lastReference = false; + taosThreadMutexLock(&shard->mutex); + + SLRUEntry *e = taosLRUEntryTableRemove(&shard->table, key, keyLen, hash); + if (e != NULL) { + assert(TAOS_LRU_ENTRY_IN_CACHE(e)); + TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); + if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { + taosLRUCacheShardLRURemove(shard, e); + + assert(shard->usage >= e->totalCharge); + shard->usage -= e->totalCharge; + lastReference = true; + } + } + + taosThreadMutexUnlock(&shard->mutex); + + if (lastReference) { + taosLRUEntryFree(e); + } +} + +static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { + SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); + + taosThreadMutexLock(&shard->mutex); + + while (shard->lru.next != &shard->lru) { + SLRUEntry *old = shard->lru.next; + assert(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); + taosLRUCacheShardLRURemove(shard, old); + taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); + TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); + assert(shard->usage >= old->totalCharge); + shard->usage -= old->totalCharge; + + taosArrayPush(lastReferenceList, &old); + } + + taosThreadMutexUnlock(&shard->mutex); + + for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { + SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); + + taosLRUEntryFree(entry); + } + + taosArrayDestroy(lastReferenceList); +} + +static bool taosLRUCacheShardRef(SLRUCacheShard *shard, LRUHandle *handle) { + SLRUEntry *e = (SLRUEntry *) handle; + taosThreadMutexLock(&shard->mutex); + + assert(TAOS_LRU_ENTRY_HAS_REFS(e)); + TAOS_LRU_ENTRY_REF(e); + + taosThreadMutexUnlock(&shard->mutex); + + return true; +} + +static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, bool eraseIfLastRef) { + if (handle == NULL) { + return false; + } + + SLRUEntry *e = (SLRUEntry *) handle; + bool lastReference = false; + + taosThreadMutexLock(&shard->mutex); + + lastReference = taosLRUEntryUnref(e); + if (lastReference && TAOS_LRU_ENTRY_IN_CACHE(e)) { + if (shard->usage > shard->capacity || eraseIfLastRef) { + assert(shard->lru.next == &shard->lru || eraseIfLastRef); + + taosLRUEntryTableRemove(&shard->table, e->keyData, e->keyLength, e->hash); + TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); + } else { + taosLRUCacheShardLRUInsert(shard, e); + + lastReference = false; + } + } + + if (lastReference && e->value) { + assert(shard->usage >= e->totalCharge); + shard->usage -= e->totalCharge; + } + + taosThreadMutexUnlock(&shard->mutex); + + if (lastReference) { + taosLRUEntryFree(e); + } + + return lastReference; +} + +static size_t taosLRUCacheShardGetUsage(SLRUCacheShard *shard) { + size_t usage = 0; + + taosThreadMutexLock(&shard->mutex); + usage = shard->usage; + taosThreadMutexUnlock(&shard->mutex); + + return usage; +} + +static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) { + size_t usage = 0; + + taosThreadMutexLock(&shard->mutex); + + assert(shard->usage >= shard->lruUsage); + usage = shard->usage - shard->lruUsage; + + taosThreadMutexUnlock(&shard->mutex); + + return usage; +} + +static void taosLRUCacheShardSetStrictCapacity(SLRUCacheShard *shard, bool strict) { + taosThreadMutexLock(&shard->mutex); + + shard->strictCapacity = strict; + + taosThreadMutexUnlock(&shard->mutex); +} + +struct SShardedCache { + uint32_t shardMask; + TdThreadMutex capacityMutex; + size_t capacity; + bool strictCapacity; + uint64_t lastId; // atomic var for last id +}; + +struct SLRUCache { + SShardedCache shardedCache; + SLRUCacheShard *shards; + int numShards; +}; + +static int getDefaultCacheShardBits(size_t capacity) { + int numShardBits = 0; + size_t minShardSize = 512 * 1024; + size_t numShards = capacity / minShardSize; + while (numShards >>= 1) { + if (++numShardBits >= 6) { + return numShardBits; + } + } + + return numShardBits; +} + +SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoolRatio) { + if (numShardBits >= 20) { + return NULL; + } + if (highPriPoolRatio < 0.0 || highPriPoolRatio > 1.0) { + return NULL; + } + SLRUCache *cache = taosMemoryCalloc(1, sizeof(SLRUCache)); + if (!cache) { + return NULL; + } + + if (numShardBits < 0) { + numShardBits = getDefaultCacheShardBits(capacity); + } + + int numShards = 1 << numShardBits; + cache->shards = taosMemoryCalloc(numShards, sizeof(SLRUCacheShard)); + if (!cache->shards) { + taosMemoryFree(cache); + + return NULL; + } + + bool strictCapacity = 1; + size_t perShard = (capacity + (numShards - 1)) / numShards; + for (int i = 0; i < numShards; ++i) { + taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits); + } + + cache->numShards = numShards; + + cache->shardedCache.shardMask = (1 << numShardBits) - 1; + cache->shardedCache.strictCapacity = strictCapacity; + cache->shardedCache.capacity = capacity; + cache->shardedCache.lastId = 1; + + taosThreadMutexInit(&cache->shardedCache.capacityMutex, NULL); + + return cache; +} + +void taosLRUCacheCleanup(SLRUCache *cache) { + if (cache) { + if (cache->shards) { + int numShards = cache->numShards; + assert(numShards > 0); + for (int i = 0; i < numShards; ++i) { + taosLRUCacheShardCleanup(&cache->shards[i]); + } + taosMemoryFree(cache->shards); + cache->shards = 0; + } + + taosThreadMutexDestroy(&cache->shardedCache.capacityMutex); + + taosMemoryFree(cache); + } +} + +LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, + _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority) { + uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); + uint32_t shardIndex = hash & cache->shardedCache.shardMask; + + return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle, priority); +} + +LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen) { + uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); + uint32_t shardIndex = hash & cache->shardedCache.shardMask; + + return taosLRUCacheShardLookup(&cache->shards[shardIndex], key, keyLen, hash); +} + +void taosLRUCacheErase(SLRUCache *cache, const void *key, size_t keyLen) { + uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); + uint32_t shardIndex = hash & cache->shardedCache.shardMask; + + return taosLRUCacheShardErase(&cache->shards[shardIndex], key, keyLen, hash); +} + +void taosLRUCacheEraseUnrefEntries(SLRUCache *cache) { + int numShards = cache->numShards; + for (int i = 0; i < numShards; ++i) { + taosLRUCacheShardEraseUnrefEntries(&cache->shards[i]); + } +} + +bool taosLRUCacheRef(SLRUCache *cache, LRUHandle *handle) { + if (handle == NULL) { + return false; + } + + uint32_t hash = ((SLRUEntry *) handle)->hash; + uint32_t shardIndex = hash & cache->shardedCache.shardMask; + + return taosLRUCacheShardRef(&cache->shards[shardIndex], handle); +} + +bool taosLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRef) { + if (handle == NULL) { + return false; + } + + uint32_t hash = ((SLRUEntry *) handle)->hash; + uint32_t shardIndex = hash & cache->shardedCache.shardMask; + + return taosLRUCacheShardRelease(&cache->shards[shardIndex], handle, eraseIfLastRef); +} + +void* taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle) { + return ((SLRUEntry*) handle)->value; +} + +size_t taosLRUCacheGetUsage(SLRUCache *cache) { + size_t usage = 0; + + for (int i = 0; i < cache->numShards; ++i) { + usage += taosLRUCacheShardGetUsage(&cache->shards[i]); + } + + return usage; +} + +size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache) { + size_t usage = 0; + + for (int i = 0; i < cache->numShards; ++i) { + usage += taosLRUCacheShardGetPinnedUsage(&cache->shards[i]); + } + + return usage; +} + +void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity) { + uint32_t numShards = cache->numShards; + size_t perShard = (capacity + (numShards = 1)) / numShards; + + taosThreadMutexLock(&cache->shardedCache.capacityMutex); + + for (int i = 0; i < numShards; ++i) { + taosLRUCacheShardSetCapacity(&cache->shards[i], perShard); + } + + cache->shardedCache.capacity = capacity; + + taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); +} + +size_t taosLRUCacheGetCapacity(SLRUCache *cache) { + size_t capacity = 0; + + taosThreadMutexLock(&cache->shardedCache.capacityMutex); + + capacity = cache->shardedCache.capacity; + + taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); + + return capacity; +} + +void taosLRUCacheSetStrictCapacity(SLRUCache *cache, bool strict) { + uint32_t numShards = cache->numShards; + + taosThreadMutexLock(&cache->shardedCache.capacityMutex); + + for (int i = 0; i < numShards; ++i) { + taosLRUCacheShardSetStrictCapacity(&cache->shards[i], strict); + } + + cache->shardedCache.strictCapacity = strict; + + taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); +} + +bool taosLRUCacheIsStrictCapacity(SLRUCache *cache) { + bool strict = false; + + taosThreadMutexLock(&cache->shardedCache.capacityMutex); + + strict = cache->shardedCache.strictCapacity; + + taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); + + return strict; +} diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0d70ab82c0..b5916e2d76 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -589,7 +589,10 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -9 %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -9 %s > nul 2>&1" % processID + else: + killCmd = "kill -9 %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -599,7 +602,10 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -TERM %s > nul 2>&1" % processID + else: + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 5a8cf562a0..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,27 +2,27 @@ #======================b1-start=============== # ---- user -./test.sh -f tsim/user/basic1.sim -./test.sh -f tsim/user/pass_alter.sim -./test.sh -f tsim/user/pass_len.sim -./test.sh -f tsim/user/user_len.sim -./test.sh -f tsim/user/privilege1.sim -./test.sh -f tsim/user/privilege2.sim +#./test.sh -f tsim/user/basic1.sim +#./test.sh -f tsim/user/pass_alter.sim +#./test.sh -f tsim/user/pass_len.sim +#./test.sh -f tsim/user/user_len.sim +#./test.sh -f tsim/user/privilege1.sim +#./test.sh -f tsim/user/privilege2.sim# -# ---- db -./test.sh -f tsim/db/create_all_options.sim -./test.sh -f tsim/db/alter_option.sim -./test.sh -f tsim/db/basic1.sim -./test.sh -f tsim/db/basic2.sim -./test.sh -f tsim/db/basic3.sim -./test.sh -f tsim/db/basic6.sim +## ---- db +#./test.sh -f tsim/db/create_all_options.sim +#./test.sh -f tsim/db/alter_option.sim +#./test.sh -f tsim/db/basic1.sim +#./test.sh -f tsim/db/basic2.sim +#./test.sh -f tsim/db/basic3.sim +#./test.sh -f tsim/db/basic6.sim ./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/error1.sim ./test.sh -f tsim/db/taosdlog.sim # ---- dnode ./test.sh -f tsim/dnode/create_dnode.sim -./test.sh -f tsim/dnode/drop_dnode_mnode.sim +./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim @@ -57,9 +57,10 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim -#./test.sh -f tsim/mnode/basic2.sim +./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic5.sim # ---- show ./test.sh -f tsim/show/basic.sim @@ -71,6 +72,7 @@ ./test.sh -f tsim/stream/basic0.sim ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim +# ./test.sh -f tsim/stream/distributeInterval0.sim # ./test.sh -f tsim/stream/session0.sim # ./test.sh -f tsim/stream/session1.sim # ./test.sh -f tsim/stream/state0.sim diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 7444511504..12babea097 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -95,7 +95,7 @@ endi if $data6_db != 345600 then # days return -1 endi -if $data7_db != 1440000,1440000,1440000 then # keep +if $data7_db != 1440000m,1440000m,1440000m then # keep return -1 endi if $data8_db != 96 then # buffer @@ -232,7 +232,7 @@ print ============== modify keep sql alter database db keep 2400 sql show databases print keep $data7_db -if $data7_db != 3456000,3456000,3456000 then +if $data7_db != 3456000m,3456000m,3456000m then return -1 endi diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 9075ebb2e8..142460f214 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -37,7 +37,7 @@ endi if $data26 != 2880 then return -1 endi -if $data27 != 14400,14400,14400 then +if $data27 != 14400m,14400m,14400m then return -1 endi #if $data28 != 32 then diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 88f0378d61..fac385a9a6 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -116,7 +116,7 @@ endi if $data6_db != 14400 then # days return -1 endi -if $data7_db != 5256000,5256000,5256000 then # keep +if $data7_db != 5256000m,5256000m,5256000m then # keep return -1 endi if $data8_db != 96 then # buffer diff --git a/tests/script/tsim/dnode/create_dnode.sim b/tests/script/tsim/dnode/create_dnode.sim index 8a28897d5a..730b80b866 100644 --- a/tests/script/tsim/dnode/create_dnode.sim +++ b/tests/script/tsim/dnode/create_dnode.sim @@ -182,16 +182,5 @@ if $rows != 15 then return -1 endi -print =============== drop dnode -#sql drop dnode 2; -#sql show dnodes; -#if $rows != 1 then -# return -1 -#endi - -#if $data00 != 1 then -# return -1 -#endi - system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim new file mode 100644 index 0000000000..1dbec2c04d --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim @@ -0,0 +1,98 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != offline then + goto step1 +endi + +print =============== step2 drop dnode 3 +sql_error drop dnode 1 +sql drop dnode 3 + +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step3: create mnode on dnode 2 +sql create mnode on dnode 2 +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes -x step3 +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $data(1)[2] != leader then + goto step3 +endi +if $data(2)[2] != follower then + goto step3 +endi + +print =============== step4: drop dnode 2 +sql drop dnode 2 + +print show dnodes; +sql show dnodes; +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 1 then + return -1 +endi +if $data00 != 1 then + return -1 +endi + +print show dnodes; +sql show mnodes +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 1 then + return -1 +endi +if $data(1)[2] != leader then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_mnode.sim b/tests/script/tsim/dnode/drop_dnode_mnode.sim deleted file mode 100644 index e0a85b9803..0000000000 --- a/tests/script/tsim/dnode/drop_dnode_mnode.sim +++ /dev/null @@ -1,52 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -sql connect - -print =============== step1 create dnode2 -sql create dnode $hostname port 7200 - -$x = 0 -step1: - $ = $x + 1 - sleep 1000 - if $x == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $data00 $data01 $data02 $data03 $data04 $data05 -print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then - return -1 -endi -if $data(1)[4] != ready then - goto step1 -endi -if $data(2)[4] != ready then - goto step1 -endi - -sql create dnode $hostname port 7300 -sql drop dnode 3 -sql_error drop dnode 1 - -print =============== step2: create mnode -sql create mnode on dnode 2 - -print =============== step3: drop dnode 3 -sql drop dnode 2 -sql show dnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -return -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_move_1_vnode.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim similarity index 90% rename from tests/script/tsim/dnode/redistribute_vgroup_replica3_move_1_vnode.sim rename to tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim index cb9f4173ac..c5cc765b88 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_move_1_vnode.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim @@ -21,7 +21,7 @@ sql create dnode $hostname port 7500 $x = 0 step1: - $ = $x + 1 + $x = $x + 1 sleep 1000 if $x == 10 then print ====> dnode not ready! @@ -69,11 +69,12 @@ sql_error redistribute vgroup 2 dnode 5 dnode 3 sql_error redistribute vgroup 2 dnode 2 dnode 3 sql_error redistribute vgroup 2 dnode 2 dnode 2 sql_error redistribute vgroup 3 dnode 2 dnode 2 +sql_error redistribute vgroup 2 dnode 2 dnode 2 dnode 3 system sh/exec.sh -n dnode5 -s start $x = 0 step2: - $ = $x + 1 + $x = $x + 1 sleep 1000 if $x == 10 then print ====> dnode not ready! @@ -112,9 +113,9 @@ $follower2 = 0 $x = 0 step3: - $ = $x + 1 + $x = $x + 1 sleep 1000 - if $x == 10 then + if $x == 60 then print ====> db not ready! return -1 endi @@ -149,31 +150,51 @@ print leader $leaderVnode print follower1 $follower1 print follower2 $follower2 +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + print =============== step32: move follower2 print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -return +sql show d1.tables +if $rows != 1 then + return -1 +endi + print =============== step33: move follower1 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql show d1.tables +if $rows != 1 then + return -1 +endi print =============== step34: move follower2 print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql show d1.tables +if $rows != 1 then + return -1 +endi print =============== step35: move follower1 print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 +sql show d1.tables +if $rows != 1 then + return -1 +endi print =============== step4: move leader -return - -print =============== step3: drop dnode 3 -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/mnode/basic2.sim b/tests/script/tsim/mnode/basic2.sim index 78558263d6..ff0101dd8e 100644 --- a/tests/script/tsim/mnode/basic2.sim +++ b/tests/script/tsim/mnode/basic2.sim @@ -92,6 +92,8 @@ sql show mnodes if $rows != 2 then return -1 endi +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 sql show users if $rows != 2 then @@ -111,6 +113,8 @@ step3: return -1 endi sql show dnodes -x step3 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 if $data(1)[4] != ready then goto step3 endi diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index dec036faaf..695e23f3ac 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -39,11 +39,9 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 -return -system sh/exec.sh -n dnode1 -s stop -x SIGKILL sql_error create mnode on dnode 4 - +$leaderExist = 0 $x = 0 step2: $x = $x + 1 @@ -52,13 +50,20 @@ step2: return -1 endi sql show mnodes -x step2 -if $data(1)[2] != leader then - goto step2 + +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data(1)[2] == leader then + $leaderExist = 1 endi -if $data(2)[2] != follower then - goto step2 +if $data(2)[2] == leader then + $leaderExist = 1 endi -if $data(3)[2] != follower then +if $data(3)[2] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then goto step2 endi @@ -70,10 +75,10 @@ if $rows != 2 then endi # wait mnode2 mnode3 recv data finish -sleep 10000 +sleep 1000 print =============== step4: stop dnode1 -system sh/exec.sh -n dnode1 -s stop +system sh/exec.sh -n dnode1 -s stop -x SIGKILL $x = 0 step4: @@ -92,13 +97,22 @@ if $rows != 2 then return -1 endi -sleep 1000 -sql show dnodes +$x = 0 +step41: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes -x step41 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 if $data(2)[4] != ready then - return -1 + goto step41 endi if $data(3)[4] != ready then - return -1 + goto step41 endi print =============== step5: stop dnode1 @@ -117,15 +131,29 @@ print $data(1)[0] $data(1)[1] $data(1)[2] print $data(2)[0] $data(2)[1] $data(2)[2] print $data(3)[0] $data(3)[1] $data(3)[2] -if $data(2)[2] != offline then - goto step5 -endi - sql show users if $rows != 2 then return -1 endi +$x = 0 +step51: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes -x step51 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data(1)[4] != ready then + goto step51 +endi +if $data(3)[4] != ready then + goto step51 +endi + print =============== step6: stop dnode1 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s stop @@ -147,6 +175,24 @@ if $rows != 2 then return -1 endi +$x = 0 +step61: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes -x step61 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data(1)[4] != ready then + goto step61 +endi +if $data(2)[4] != ready then + goto step61 +endi + system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop system sh/exec.sh -n dnode3 -s stop diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index 88deb5af89..41524946be 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -117,7 +117,7 @@ if $data(3)[3] != ready then endi print =============== step5: drop mnode 3 and stop dnode3 -system sh/exec.sh -n dnode3 -s stop +system sh/exec.sh -n dnode3 -s stop -x SIGKILL sql_error drop mnode on dnode 3 $x = 0 diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim new file mode 100644 index 0000000000..399dced169 --- /dev/null +++ b/tests/script/tsim/mnode/basic5.sim @@ -0,0 +1,349 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== step1: create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 5 then + return -1 + endi +sql show dnodes +if $data(1)[4] != ready then + goto step1 +endi + +print =============== step2: create dnodes - with error +sql_error create mnode on dnode 1; +sql_error create mnode on dnode 2; +sql_error create mnode on dnode 3; +sql_error create mnode on dnode 4; +sql_error create mnode on dnode 5; +sql_error create mnode on dnode 6; + +print =============== step3: create mnode 2 and 3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 5 then + return -1 + endi +sql show dnodes +if $data(2)[4] != ready then + goto step3 +endi +if $data(3)[4] != ready then + goto step3 +endi +if $data(4)[4] != ready then + goto step3 +endi + +sql create mnode on dnode 2 +sql create mnode on dnode 3 + +$x = 0 +step31: + $x = $x + 1 + sleep 1000 + if $x == 50 then + return -1 + endi +sql show mnodes +if $data(1)[2] != leader then + goto step31 +endi +if $data(2)[2] != follower then + goto step31 +endi +if $data(3)[2] != follower then + goto step31 +endi + +print =============== step4: create dnodes - with error +sql_error create mnode on dnode 1 +sql_error create mnode on dnode 2; +sql_error create mnode on dnode 3; +sql_error create mnode on dnode 4; +sql_error create mnode on dnode 5; +sql_error create mnode on dnode 6; + +print =============== step5: drop mnodes - with error +sql_error drop mnode on dnode 1 +sql_error drop mnode on dnode 4 +sql_error drop mnode on dnode 5 +sql_error drop mnode on dnode 6 + +system sh/exec.sh -n dnode2 -s stop -x SIGKILL +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $data(1)[4] != ready then + goto step5 +endi +if $data(2)[4] != offline then + goto step5 +endi +if $data(3)[4] != ready then + goto step5 +endi +if $data(4)[4] != ready then + goto step5 +endi + +sql_error drop mnode on dnode 2 + +system sh/exec.sh -n dnode2 -s start +$x = 0 +step51: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $data(1)[4] != ready then + goto step51 +endi +if $data(2)[4] != ready then + goto step51 +endi +if $data(3)[4] != ready then + goto step51 +endi +if $data(4)[4] != ready then + goto step51 +endi + +print =============== step6: stop mnode1 +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sql_error drop mnode on dnode 1 + +$x = 0 +step61: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +$leaderNum = 0 +if $data(2)[2] == leader then + $leaderNum = 1 +endi +if $data(3)[2] == leader then + $leaderNum = 1 +endi +if $leaderNum != 1 then + goto step61 +endi + +print =============== step7: start mnode1 and wait it online +system sh/exec.sh -n dnode1 -s start + +$x = 0 +step71: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $data(1)[4] != ready then + goto step71 +endi +if $data(2)[4] != ready then + goto step71 +endi +if $data(3)[4] != ready then + goto step71 +endi +if $data(4)[4] != ready then + goto step71 +endi + +print =============== step8: stop mnode1 and drop it +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sql_error drop mnode on dnode 1 + +$x = 0 +step81: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +$leaderNum = 0 +if $data(1)[2] == leader then + $leaderNum = 1 +endi +if $data(2)[2] == leader then + $leaderNum = 1 +endi +if $data(3)[2] == leader then + $leaderNum = 1 +endi +if $leaderNum != 1 then + goto step81 +endi +if $data(1)[3] != dropping then + goto step81 +endi + +print =============== step9: start mnode1 and wait it dropped +system sh/exec.sh -n dnode1 -s start + +$x = 0 +step91: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $data(1)[4] != ready then + goto step91 +endi +if $data(2)[4] != ready then + goto step91 +endi +if $data(3)[4] != ready then + goto step91 +endi +if $data(4)[4] != ready then + goto step91 +endi + +$x = 0 +step92: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +$leaderNum = 0 +if $data(1)[2] == leader then + $leaderNum = 1 +endi +if $data(2)[2] == leader then + $leaderNum = 1 +endi +if $data(3)[2] == leader then + $leaderNum = 1 +endi +if $leaderNum != 1 then + goto step92 +endi +if $rows != 2 then + goto step92 +endi + +print =============== stepa: create mnode1 again +sql create mnode on dnode 1 +$x = 0 +stepa: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +$leaderNum = 0 +if $data(2)[2] == leader then + $leaderNum = 1 +endi +if $data(3)[2] == leader then + $leaderNum = 1 +endi +if $data(3)[2] == leader then + $leaderNum = 1 +endi +if $leaderNum == 0 then + goto stepa +endi +if $leaderNum != 1 then + return -1 +endi + +$x = 0 +stepb: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto stepb +endi +if $data(2)[4] != ready then + goto stepb +endi +if $data(3)[4] != ready then + goto stepb +endi +if $data(4)[4] != ready then + goto stepb +endi + +system sh/exec.sh -n dnode1 -s stop +system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode3 -s stop +system sh/exec.sh -n dnode4 -s stop diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim new file mode 100644 index 0000000000..f4f3e04f0a --- /dev/null +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -0,0 +1,176 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create dnode $hostname2 port 7200 + +system sh/exec.sh -n dnode2 -s start + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); + +sql insert into ts3 values(1648791213001,1,12,3,1.0); +sql insert into ts4 values(1648791213001,1,12,3,1.0); + +sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts3 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts4 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts1 values(1648791223002,2,2,3,1.1); +sql insert into ts1 values(1648791233003,3,2,3,2.1); +sql insert into ts2 values(1648791243004,4,2,43,73.1); +sql insert into ts1 values(1648791213002,24,22,23,4.1); +sql insert into ts1 values(1648791243005,4,20,3,3.1); +sql insert into ts2 values(1648791243006,4,2,3,3.1) (1648791243007,4,2,3,3.1) ; +sql insert into ts1 values(1648791243008,4,2,30,3.1) (1648791243009,4,2,3,3.1) (1648791243010,4,2,3,3.1) ; +sql insert into ts2 values(1648791243011,4,2,3,3.1) (1648791243012,34,32,33,3.1) (1648791243013,4,2,3,3.1) (1648791243014,4,2,13,3.1); +sql insert into ts1 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; +sql insert into ts2 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) (1648791233004,13,12,13,2.1) ; +sql insert into ts1 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; + +sql insert into ts3 values(1648791223002,2,2,3,1.1); +sql insert into ts4 values(1648791233003,3,2,3,2.1); +sql insert into ts3 values(1648791243004,4,2,43,73.1); +sql insert into ts4 values(1648791213002,24,22,23,4.1); +sql insert into ts3 values(1648791243005,4,20,3,3.1); +sql insert into ts4 values(1648791243006,4,2,3,3.1) (1648791243007,4,2,3,3.1) ; +sql insert into ts3 values(1648791243008,4,2,30,3.1) (1648791243009,4,2,3,3.1) (1648791243010,4,2,3,3.1) ; +sql insert into ts4 values(1648791243011,4,2,3,3.1) (1648791243012,34,32,33,3.1) (1648791243013,4,2,3,3.1) (1648791243014,4,2,13,3.1); +sql insert into ts3 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; +sql insert into ts4 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) (1648791233004,13,12,13,2.1) ; +sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; + +$loop_count = 0 +loop1: +sql select * from streamtST1; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 8 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop1 +endi + +if $data03 != 4 then + print ======$data03 + return -1 +endi + +if $data04 != 52 then + print ======$data04 + return -1 +endi + +if $data05 != 13 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 6 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 6 then + print =====data12=$data12 + goto loop1 +endi + +if $data13 != 92 then + print ======$data13 + return -1 +endi + +if $data14 != 22 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 4 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 4 then + print =====data22=$data22 + goto loop1 +endi + +if $data23 != 32 then + print ======$data23 + return -1 +endi + +if $data24 != 12 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 30 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 30 then + print =====data32=$data32 + goto loop1 +endi + +if $data33 != 180 then + print ======$data33 + return -1 +endi + +if $data34 != 42 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index c021e14de7..2a737578bc 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -231,8 +231,10 @@ sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams3 trigger at_once watermark 1d into streamt3 as select _wstartts, min(b), a,c from t1 session(ts,10s); sql create stream streams4 trigger at_once watermark 1d into streamt4 as select _wstartts, max(b), a,c from t1 session(ts,10s); -sql create stream streams5 trigger at_once watermark 1d into streamt5 as select _wstartts, max(b), a,c from t1 session(ts,10s); -sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, max(b), a,c from t1 session(ts,10s); +sql create stream streams5 trigger at_once watermark 1d into streamt5 as select _wstartts, top(b,3), a,c from t1 session(ts,10s); +sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, bottom(b,3), a,c from t1 session(ts,10s); +sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s); +sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s); sql insert into t1 values(1648791213001,1,1,1,1.0); sql insert into t1 values(1648791213002,2,3,2,3.4); sql insert into t1 values(1648791213003,4,9,3,4.8); @@ -279,4 +281,16 @@ if $rows == 0 then goto loop3 endi +sql select * from streamt7; +if $rows == 0 then + print ======$rows + goto loop3 +endi + +sql select * from streamt8; +if $rows == 0 then + print ======$rows + goto loop3 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index e8c3544152..c92efb403c 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -122,10 +122,10 @@ class TDTestCase: for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: for j in [1,2,3]: - tdSql.error(f'alter table {stbname} modify tag t{j} {i}') + tdSql.error(f'alter stable {stbname} modify tag t{j} {i}') for i in ['int','unsigned int','float','binary(10)','nchar(10)']: - tdSql.error(f'alter table {stbname} modify tag t8 {i}') - tdSql.error(f'alter table {stbname} modify tag t4 int') + tdSql.error(f'alter stable {stbname} modify tag t8 {i}') + tdSql.error(f'alter stable {stbname} modify tag t4 int') tdSql.execute(f'drop database {dbname}') def run(self): diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index 53e3793dcf..3c0def86e4 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -145,10 +145,17 @@ class TDTestCase: tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21') tdSql.query(f'describe {dbname}.{tbname}') tdSql.checkData(1,0,'c21') + # !bug TD-16423 + # tdSql.error(f'select c1 from {dbname}.{tbname}') + # tdSql.query(f'select c21 from {dbname}.{tbname}') + # tdSql.checkData(0,1,1) tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1') tdSql.query(f'describe {dbname}.{tbname}') tdSql.checkData(1,0,'c1') - + # !bug TD-16423 + # tdSql.error(f'select c1 from {dbname}.{tbname}') + # tdSql.query(f'select c1 from {dbname}.{tbname}') + # tdSql.checkData(0,1,1) tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint') tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double') tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int') @@ -158,9 +165,109 @@ class TDTestCase: tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool') tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)') tdSql.execute(f'drop database {dbname}') + def alter_stb_column_check(self): + dbname = self.get_long_name(length=10, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + stbname = self.get_long_name(length=3, mode="letters") + tbname = self.get_long_name(length=3, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ + c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') + tdSql.execute(f'alter table {stbname} add column c14 int') + tdSql.query(f'select c14 from {stbname}') + tdSql.checkRows(1) + tdSql.execute(f'alter table {stbname} add column `c15` int') + tdSql.query(f'select c15 from {stbname}') + tdSql.checkRows(1) + tdSql.query(f'describe {stbname}') + tdSql.checkRows(17) + tdSql.execute(f'alter table {stbname} drop column c14') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(16) + tdSql.execute(f'alter table {stbname} drop column `c15`') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(15) + tdSql.execute(f'alter table {stbname} modify column c12 binary(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(12,2,30) + tdSql.execute(f'alter table {stbname} modify column `c12` binary(35)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(12,2,35) + tdSql.error(f'alter table {stbname} modify column `c12` binary(34)') + tdSql.execute(f'alter table {stbname} modify column c13 nchar(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(13,2,30) + tdSql.error(f'alter table {stbname} modify column c13 nchar(29)') + tdSql.error(f'alter table {stbname} rename column c1 c21') + tdSql.error(f'alter table {stbname} modify column c1 int') + tdSql.error(f'alter table {stbname} modify column c4 int') + tdSql.error(f'alter table {stbname} modify column c8 int') + tdSql.error(f'alter table {stbname} modify column c1 unsigned int') + tdSql.error(f'alter table {stbname} modify column c9 double') + tdSql.error(f'alter table {stbname} modify column c10 float') + tdSql.error(f'alter table {stbname} modify column c11 int') + tdSql.execute(f'drop database {dbname}') + def alter_stb_tag_check(self): + dbname = self.get_long_name(length=10, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + stbname = self.get_long_name(length=3, mode="letters") + tbname = self.get_long_name(length=3, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \ + t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ') + tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') + tdSql.execute(f'insert into {tbname} values(now,1)') + + tdSql.execute(f'alter table {stbname} add tag t14 int') + tdSql.query(f'select t14 from {stbname}') + tdSql.checkRows(1) + tdSql.execute(f'alter table {stbname} add tag `t15` int') + tdSql.query(f'select t14 from {stbname}') + tdSql.checkRows(1) + tdSql.query(f'describe {stbname}') + tdSql.checkRows(18) + tdSql.execute(f'alter table {stbname} drop tag t14') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(17) + tdSql.execute(f'alter table {stbname} drop tag `t15`') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(16) + tdSql.execute(f'alter table {stbname} modify tag t12 binary(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(14,2,30) + tdSql.execute(f'alter table {stbname} modify tag `t12` binary(35)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(14,2,35) + tdSql.error(f'alter table {stbname} modify tag `t12` binary(34)') + tdSql.execute(f'alter table {stbname} modify tag t13 nchar(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(15,2,30) + tdSql.error(f'alter table {stbname} modify tag t13 nchar(29)') + tdSql.execute(f'alter table {stbname} rename tag t1 t21') + tdSql.query(f'describe {stbname}') + tdSql.checkData(3,0,'t21') + tdSql.execute(f'alter table {stbname} rename tag `t21` t1') + tdSql.query(f'describe {stbname}') + tdSql.checkData(3,0,'t1') + + for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: + for j in [1,2,3]: + tdSql.error(f'alter table {stbname} modify tag t{j} {i}') + for i in ['int','unsigned int','float','binary(10)','nchar(10)']: + tdSql.error(f'alter table {stbname} modify tag t8 {i}') + tdSql.error(f'alter table {stbname} modify tag t4 int') + tdSql.execute(f'drop database {dbname}') def run(self): self.alter_tb_tag_check() self.alter_ntb_column_check() + self.alter_stb_column_check() + self.alter_stb_tag_check() def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 0ddeba4665..10e81892ef 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -23,6 +23,10 @@ from util.sql import * import threading from util.types import TDSmlProtocolType, TDSmlTimestampType from util.common import tdCom +import platform +import io +if platform.system().lower() == 'windows': + sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') class TDTestCase: def init(self, conn, logSql): diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index 23404330ed..649f0101af 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -22,6 +22,10 @@ from util.sql import * from util.common import tdCom from util.types import TDSmlProtocolType, TDSmlTimestampType import threading +import platform +import io +if platform.system().lower() == 'windows': + sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') class TDTestCase: def init(self, conn, logSql): diff --git a/tests/system-test/2-query/To_iso8601.py b/tests/system-test/2-query/To_iso8601.py index 57bcca638c..da7b7e272f 100644 --- a/tests/system-test/2-query/To_iso8601.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -1,9 +1,10 @@ +import time from time import sleep from util.log import * from util.sql import * from util.cases import * - +import os @@ -12,28 +13,47 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - - def run(self): # sourcery skip: extract-duplicate-method - tdSql.prepare() - # get system timezone - today_date = datetime.datetime.strptime( - datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") + self.rowNum = 10 + self.ts = 1640966400000 # 2022-1-1 00:00:00.000 + def check_customize_param_ms(self): + time_zone = time.strftime('%z') + tdSql.execute('create database db1 precision "ms"') + tdSql.execute('use db1') + tdSql.execute('create table if not exists ntb(ts timestamp, c1 int, c2 timestamp)') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d)" + % (self.ts + i, i + 1, self.ts + i)) + tdSql.query('select to_iso8601(ts) from ntb') + for i in range(self.rowNum): + tdSql.checkEqual(tdSql.queryResult[i][0],f'2022-01-01T00:00:00.00{i}{time_zone}') + + timezone_list = ['+0000','+0100','+0200','+0300','+0330','+0400','+0500','+0530','+0600','+0700','+0800','+0900','+1000','+1100','+1200',\ + '+00','+01','+02','+03','+04','+05','+06','+07','+08','+09','+10','+11','+12',\ + '+00:00','+01:00','+02:00','+03:00','+03:30','+04:00','+05:00','+05:30','+06:00','+07:00','+08:00','+09:00','+10:00','+11:00','+12:00',\ + '-0000','-0100','-0200','-0300','-0400','-0500','-0600','-0700','-0800','-0900','-1000','-1100','-1200',\ + '-00','-01','-02','-03','-04','-05','-06','-07','-08','-09','-10','-11','-12',\ + '-00:00','-01:00','-02:00','-03:00','-04:00','-05:00','-06:00','-07:00','-08:00','-09:00','-10:00','-11:00','-12:00',\ + 'z','Z'] + for j in timezone_list: + tdSql.query(f'select to_iso8601(ts,"{j}") from ntb') + for i in range(self.rowNum): + tdSql.checkEqual(tdSql.queryResult[i][0],f'2022-01-01T00:00:00.00{i}{j}') + + error_param_list = [0,100.5,'a','!'] + for i in error_param_list: + tdSql.error(f'select to_iso8601(ts,"{i}") from ntb') + #! bug TD-16372:对于错误的时区，缺少校验 + error_timezone_param = ['+13','-13','+1300','-1300','+0001','-0001','-0330','-0530'] + for i in error_timezone_param: + tdSql.error(f'select to_iso8601(ts,"{i}") from ntb') + + def check_base_function(self): + tdSql.prepare() tdLog.printNoPrefix("==========step1:create tables==========") - tdSql.execute( - '''create table if not exists ntb - (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) - ''' - ) - tdSql.execute( - '''create table if not exists stb - (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) - ''' - ) - tdSql.execute( - '''create table if not exists stb_1 using stb tags(100) - ''' - ) + tdSql.execute('create table if not exists ntb(ts timestamp, c1 int, c2 float,c3 double,c4 timestamp)') + tdSql.execute('create table if not exists stb(ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int)') + tdSql.execute('create table if not exists stb_1 using stb tags(100)') tdLog.printNoPrefix("==========step2:insert data==========") tdSql.execute('insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') @@ -48,12 +68,9 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query("select to_iso8601(ts) from ntb where ts=today()") tdSql.checkRows(1) - # tdSql.checkData(0,0,10) - for i in range(1,10): + for i in range(0,3): tdSql.query("select to_iso8601(1) from ntb") - tdSql.checkData(0,0,"1970-01-01T08:00:01+0800") - i+=1 - sleep(0.2) + tdSql.checkData(i,0,"1970-01-01T08:00:01+0800") tdSql.checkRows(3) tdSql.query("select to_iso8601(ts) from ntb") tdSql.checkRows(3) @@ -67,53 +84,34 @@ class TDTestCase: tdSql.error("select to_iso8601(timezone()) from ntb") tdSql.error("select to_iso8601('abc') from ntb") - tdSql.query("select to_iso8601(today()) *null from ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) +null from ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) -null from ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) /null from ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - - tdSql.query("select to_iso8601(today()) *null from db.ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) +null from db.ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) -null from db.ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) /null from db.ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - # tdSql.query("select to_iso8601(-1) from ntb") + for i in ['+','-','*','/']: + tdSql.query(f"select to_iso8601(today()) {i}null from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.query(f"select to_iso8601(today()) {i}null from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) tdSql.query("select to_iso8601(9223372036854775807) from ntb") tdSql.checkRows(3) - # bug TD-14896 + # bug TD-15207 # tdSql.query("select to_iso8601(10000000000) from ntb") # tdSql.checkData(0,0,None) # tdSql.query("select to_iso8601(-1) from ntb") # tdSql.checkRows(3) # tdSql.query("select to_iso8601(-10000000000) from ntb") # tdSql.checkData(0,0,None) - tdSql.error("select to_iso8601(1.5) from ntb") - tdSql.error("select to_iso8601(1.5) from db.ntb") - tdSql.error("select to_iso8601('a') from ntb") - tdSql.error("select to_iso8601(c2) from ntb") + err_param = [1.5,'a','c2'] + for i in err_param: + tdSql.error(f"select to_iso8601({i}) from ntb") + tdSql.error(f"select to_iso8601({i}) from db.ntb") + tdSql.query("select to_iso8601(now) from stb") + tdSql.checkRows(3) tdSql.query("select to_iso8601(now()) from stb") tdSql.checkRows(3) - for i in range(1,10): - tdSql.query("select to_iso8601(1) from stb") - tdSql.checkData(0,0,"1970-01-01T08:00:01+0800") - i+=1 - sleep(0.2) + tdSql.query("select to_iso8601(1) from stb") + for i in range(0,3): + tdSql.checkData(i,0,"1970-01-01T08:00:01+0800") tdSql.checkRows(3) tdSql.query("select to_iso8601(ts) from stb") tdSql.checkRows(3) @@ -121,37 +119,17 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select to_iso8601(ts)+'a' from stb ") tdSql.checkRows(3) + for i in ['+','-','*','/']: + tdSql.query(f"select to_iso8601(today()) {i}null from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.query(f"select to_iso8601(today()) {i}null from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) *null from stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) +null from stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) -null from stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) /null from stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) *null from db.stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) +null from db.stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) -null from db.stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - tdSql.query("select to_iso8601(today()) /null from db.stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,None) - - # bug TD-14896 - # tdSql.query("select to_iso8601(-1) from ntb") - # tdSql.checkRows(3) - - + def run(self): # sourcery skip: extract-duplicate-method + self.check_base_function() + self.check_customize_param_ms() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index a3e976b490..d9c37be996 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -65,6 +65,60 @@ class TDTestCase: ''' ) + + def prepare_tag_datas(self): + # prepare datas + tdSql.execute("create database if not exists testdb keep 3650 days 1000") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + def check_result_auto(self ,origin_query , abs_query): abs_result = tdSql.getResult(abs_query) origin_result = tdSql.getResult(origin_query) @@ -95,6 +149,7 @@ class TDTestCase: tdLog.info("abs value check pass , it work as expected ,sql is \"%s\" "%abs_query ) def test_errors(self): + tdSql.execute("use testdb") error_sql_lists = [ "select abs from t1", # "select abs(-+--+c1) from t1", @@ -129,11 +184,16 @@ class TDTestCase: tdSql.error(error_sql) def support_types(self): + tdSql.execute("use testdb") type_error_sql_lists = [ "select abs(ts) from t1" , + "select abs(t0) from t1" , "select abs(c7) from t1", "select abs(c8) from t1", "select abs(c9) from t1", + "select abs(t7) from t1", + "select abs(t8) from t1", + "select abs(t9) from t1", "select abs(ts) from ct1" , "select abs(c7) from ct1", "select abs(c8) from ct1", @@ -171,6 +231,13 @@ class TDTestCase: "select abs(c5) from t1", "select abs(c6) from t1", + "select abs(t1) from ct1", + "select abs(t2) from ct1", + "select abs(t3) from ct1", + "select abs(t4) from ct1", + "select abs(t5) from ct1", + "select abs(t6) from ct1", + "select abs(c1) from ct1", "select abs(c2) from ct1", "select abs(c3) from ct1", @@ -447,6 +514,38 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") + def test_tag_compute_for_scalar_function(self): + + tdSql.execute("use testdb") + + self.check_result_auto( "select c1, t2, t3 , t4, t5 from ct4 ", "select (c1), abs(t2) ,abs(t3), abs(t4), abs(t5) from ct4") + self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from ct4 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from ct4") + self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from stb1 order by t1 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from stb1 order by t1") + + # bug need fix + + # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash + tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") + tdSql.checkRows(13) + # tdSql.query("select t1 from stb1 where t1 >0 ") + # tdSql.checkRows(3) + # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") + # tdSql.checkData(0,0,61) + # tdSql.query("select distinct(c1) ,t1 from stb1") + # tdSql.checkRows(11) + # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") + # tdSql.checkData(0,3,33333) + + # tag filter with abs function + # tdSql.query("select t1 from stb1 where abs(t1)=1") + # tdSql.checkRows(1) + tdSql.query("select t1 from stb1 where abs(c1+t1)=1") + tdSql.checkRows(1) + # tdSql.query("select t1 from stb1 where abs(t1+c1)=1") + # tdSql.checkRows(1) + tdSql.query("select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -454,6 +553,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:create table ==============") self.prepare_datas() + self.prepare_tag_datas() tdLog.printNoPrefix("==========step2:test errors ==============") @@ -475,6 +575,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step6: tag coumpute query ============") + + self.test_tag_compute_for_scalar_function() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 5c2de5ea14..b7fa9a0cbb 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -83,6 +83,8 @@ class TDTestCase: tdSql.query(f"select {col} {alias} from {table_expr} {pre_condition}") pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') print("data is ", pre_data) pre_csum = np.cumsum(pre_data) tdSql.query(self.csum_query_form( @@ -124,6 +126,8 @@ class TDTestCase: tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): + pre_result = np.array(pre_result, dtype = 'int64') pre_csum = np.cumsum(pre_result)[offset_val:] tdSql.query(self.csum_query_form( col=col, alias=alias, table_expr=table_expr, condition=condition diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 325bd2bc8e..8edc96ed81 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -83,6 +83,8 @@ class TDTestCase: tdSql.query(f"select {col} {alias} from {table_expr} {pre_condition}") pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') pre_diff = np.diff(pre_data) # trans precision for data tdSql.query(self.diff_query_form( @@ -127,6 +129,8 @@ class TDTestCase: tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): + pre_result = np.array(pre_result, dtype = 'int64') pre_diff = np.diff(pre_result)[offset_val:] tdSql.query(self.diff_query_form( col=col, alias=alias, table_expr=table_expr, condition=condition diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index 13d2b4d420..fa2d0f47a4 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -245,6 +245,8 @@ class TDTestCase: tdSql.query(f"select {col} {alias} from {table_expr} {pre_condition}") pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') pre_mavg = np.convolve(pre_data, np.ones(k), "valid")/k tdSql.query(self.mavg_query_form( sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, @@ -291,6 +293,8 @@ class TDTestCase: # print(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") if not tdSql.queryResult: pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): + pre_result = np.array(pre_result, dtype = 'int64') pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k tdSql.query(self.mavg_query_form( diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index d6ac4d4208..3d9efea938 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -196,11 +196,13 @@ class TDTestCase: showMsg = 1 showRow = 1 - shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -312,12 +314,13 @@ class TDTestCase: pollDelay = 100 showMsg = 1 showRow = 1 - - shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) if (platform.system().lower() == 'windows'): - shellCmd += "> nul 2>&1 &" + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -448,11 +451,13 @@ class TDTestCase: showMsg = 1 showRow = 1 - shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py index 70d02c4e29..fd793fd841 100644 --- a/tests/system-test/7-tmq/db.py +++ b/tests/system-test/7-tmq/db.py @@ -98,15 +98,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index 279518d283..b2c569e31e 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -81,15 +81,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index b0b8b06076..c9f256ed74 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -81,15 +81,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index 9af78ce6c3..44c24eb616 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -81,15 +81,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -291,8 +295,12 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] + tdSql.query("select count(*) from %s.%s" %(parameterDict['dbName'], parameterDict['stbName'])) + countOfStb = tdSql.getData(0,0) + print ("====total rows of stb: %d"%countOfStb) + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - if totalConsumeRows != expectrowcnt: + if totalConsumeRows < expectrowcnt: tdLog.exit("tmq consume rows error!") tdLog.info("again start consume processer") @@ -361,7 +369,10 @@ class TDTestCase: time.sleep(2) tdLog.info("pkill consume processor") - os.system('pkill tmq_sim') + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') expectRows = 0 resultList = self.selectConsumeResult(expectRows) @@ -433,7 +444,10 @@ class TDTestCase: time.sleep(5) tdLog.info("pkill consume processor") - os.system('pkill tmq_sim') + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') expectRows = 0 resultList = self.selectConsumeResult(expectRows) diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 9f308abd7c..4f70340b5a 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -93,15 +93,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index f7e56b4550..65eaab897d 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -93,15 +93,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index 4098d151d1..90d77dba0d 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -93,15 +93,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index 45feb21019..74caa139f1 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -93,15 +93,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 81105f5352..e6eaa17564 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -93,15 +93,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index a6f1cab4a4..d8a9ca95b0 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -93,15 +93,19 @@ class TDTestCase: return resultList def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): - shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index c8084c3ad5..8a11ad42d4 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -1,103 +1,102 @@ -@REM python3 .\test.py -f 0-others\taosShell.py -@REM python3 .\test.py -f 0-others\taosShellError.py +python3 .\test.py -f 0-others\taosShell.py +python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py python3 .\test.py -f 0-others\telemetry.py python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py python3 .\test.py -f 0-others\udf_create.py -@REM python3 .\test.py -f 0-others\udf_restart_taosd.py -@REM python3 .\test.py -f 0-others\cachelast.py +python3 .\test.py -f 0-others\udf_restart_taosd.py +python3 .\test.py -f 0-others\cachelast.py -@REM python3 .\test.py -f 0-others\user_control.py -@REM python3 .\test.py -f 0-others\fsync.py +python3 .\test.py -f 0-others\user_control.py +python3 .\test.py -f 0-others\fsync.py -@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py -@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py -@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py +python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py +python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py +python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py @REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py -@REM python3 .\test.py -f 1-insert\alter_stable.py -@REM python3 .\test.py -f 1-insert\alter_table.py -@REM python3 .\test.py -f 2-query\between.py -@REM python3 .\test.py -f 2-query\distinct.py -@REM python3 .\test.py -f 2-query\varchar.py -@REM python3 .\test.py -f 2-query\ltrim.py -@REM python3 .\test.py -f 2-query\rtrim.py -@REM python3 .\test.py -f 2-query\length.py -@REM python3 .\test.py -f 2-query\char_length.py -@REM python3 .\test.py -f 2-query\upper.py -@REM python3 .\test.py -f 2-query\lower.py -@REM python3 .\test.py -f 2-query\join.py -@REM python3 .\test.py -f 2-query\join2.py -@REM python3 .\test.py -f 2-query\cast.py -@REM python3 .\test.py -f 2-query\union.py -@REM python3 .\test.py -f 2-query\union1.py -@REM python3 .\test.py -f 2-query\concat.py -@REM python3 .\test.py -f 2-query\concat2.py -@REM python3 .\test.py -f 2-query\concat_ws.py -@REM python3 .\test.py -f 2-query\concat_ws2.py -@REM python3 .\test.py -f 2-query\check_tsdb.py -@REM python3 .\test.py -f 2-query\spread.py +python3 .\test.py -f 1-insert\alter_stable.py +python3 .\test.py -f 1-insert\alter_table.py +python3 .\test.py -f 2-query\between.py +python3 .\test.py -f 2-query\distinct.py +python3 .\test.py -f 2-query\varchar.py +python3 .\test.py -f 2-query\ltrim.py +python3 .\test.py -f 2-query\rtrim.py +python3 .\test.py -f 2-query\length.py +python3 .\test.py -f 2-query\char_length.py +python3 .\test.py -f 2-query\upper.py +python3 .\test.py -f 2-query\lower.py +python3 .\test.py -f 2-query\join.py +python3 .\test.py -f 2-query\join2.py +python3 .\test.py -f 2-query\cast.py +python3 .\test.py -f 2-query\union.py +python3 .\test.py -f 2-query\union1.py +python3 .\test.py -f 2-query\concat.py +python3 .\test.py -f 2-query\concat2.py +python3 .\test.py -f 2-query\concat_ws.py +python3 .\test.py -f 2-query\concat_ws2.py +python3 .\test.py -f 2-query\check_tsdb.py +python3 .\test.py -f 2-query\spread.py @REM python3 .\test.py -f 2-query\hyperloglog.py +python3 .\test.py -f 2-query\timezone.py +python3 .\test.py -f 2-query\Now.py +python3 .\test.py -f 2-query\Today.py +python3 .\test.py -f 2-query\max.py +python3 .\test.py -f 2-query\min.py +python3 .\test.py -f 2-query\count.py +python3 .\test.py -f 2-query\last.py +python3 .\test.py -f 2-query\first.py +python3 .\test.py -f 2-query\To_iso8601.py +python3 .\test.py -f 2-query\To_unixtimestamp.py +python3 .\test.py -f 2-query\timetruncate.py +python3 .\test.py -f 2-query\diff.py +python3 .\test.py -f 2-query\Timediff.py -@REM python3 .\test.py -f 2-query\timezone.py -@REM python3 .\test.py -f 2-query\Now.py -@REM python3 .\test.py -f 2-query\Today.py -@REM python3 .\test.py -f 2-query\max.py -@REM python3 .\test.py -f 2-query\min.py -@REM python3 .\test.py -f 2-query\count.py -@REM python3 .\test.py -f 2-query\last.py -@REM python3 .\test.py -f 2-query\first.py -@REM python3 .\test.py -f 2-query\To_iso8601.py -@REM python3 .\test.py -f 2-query\To_unixtimestamp.py -@REM python3 .\test.py -f 2-query\timetruncate.py -@REM python3 .\test.py -f 2-query\diff.py -@REM python3 .\test.py -f 2-query\Timediff.py - -@REM python3 .\test.py -f 2-query\top.py -@REM python3 .\test.py -f 2-query\bottom.py -@REM python3 .\test.py -f 2-query\percentile.py -@REM python3 .\test.py -f 2-query\apercentile.py -@REM python3 .\test.py -f 2-query\abs.py -@REM python3 .\test.py -f 2-query\ceil.py -@REM python3 .\test.py -f 2-query\floor.py -@REM python3 .\test.py -f 2-query\round.py -@REM python3 .\test.py -f 2-query\log.py -@REM python3 .\test.py -f 2-query\pow.py -@REM python3 .\test.py -f 2-query\sqrt.py -@REM python3 .\test.py -f 2-query\sin.py -@REM python3 .\test.py -f 2-query\cos.py -@REM python3 .\test.py -f 2-query\tan.py -@REM python3 .\test.py -f 2-query\arcsin.py -@REM python3 .\test.py -f 2-query\arccos.py -@REM python3 .\test.py -f 2-query\arctan.py -@REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py +python3 .\test.py -f 2-query\top.py +python3 .\test.py -f 2-query\bottom.py +python3 .\test.py -f 2-query\percentile.py +python3 .\test.py -f 2-query\apercentile.py +python3 .\test.py -f 2-query\abs.py +python3 .\test.py -f 2-query\ceil.py +python3 .\test.py -f 2-query\floor.py +python3 .\test.py -f 2-query\round.py +python3 .\test.py -f 2-query\log.py +python3 .\test.py -f 2-query\pow.py +python3 .\test.py -f 2-query\sqrt.py +python3 .\test.py -f 2-query\sin.py +python3 .\test.py -f 2-query\cos.py +python3 .\test.py -f 2-query\tan.py +python3 .\test.py -f 2-query\arcsin.py +python3 .\test.py -f 2-query\arccos.py +python3 .\test.py -f 2-query\arctan.py +python3 .\test.py -f 2-query\query_cols_tags_and_or.py @REM # python3 .\test.py -f 2-query\nestedQuery.py @REM # TD-15983 subquery output duplicate name column. @REM # Please Xiangyang Guo modify the following script @REM # python3 .\test.py -f 2-query\nestedQuery_str.py -@REM python3 .\test.py -f 2-query\avg.py -@REM python3 .\test.py -f 2-query\elapsed.py -@REM python3 .\test.py -f 2-query\csum.py -@REM python3 .\test.py -f 2-query\mavg.py -@REM python3 .\test.py -f 2-query\diff.py -@REM python3 .\test.py -f 2-query\sample.py -@REM python3 .\test.py -f 2-query\function_diff.py -@REM python3 .\test.py -f 2-query\unique.py -@REM python3 .\test.py -f 2-query\stateduration.py -@REM python3 .\test.py -f 2-query\function_stateduration.py -@REM python3 .\test.py -f 2-query\statecount.py +python3 .\test.py -f 2-query\avg.py +python3 .\test.py -f 2-query\elapsed.py +python3 .\test.py -f 2-query\csum.py +python3 .\test.py -f 2-query\mavg.py +python3 .\test.py -f 2-query\diff.py +python3 .\test.py -f 2-query\sample.py +python3 .\test.py -f 2-query\function_diff.py +python3 .\test.py -f 2-query\unique.py +python3 .\test.py -f 2-query\stateduration.py +python3 .\test.py -f 2-query\function_stateduration.py +python3 .\test.py -f 2-query\statecount.py -@REM python3 .\test.py -f 7-tmq\basic5.py -@REM python3 .\test.py -f 7-tmq\subscribeDb.py -@REM python3 .\test.py -f 7-tmq\subscribeDb0.py -@REM python3 .\test.py -f 7-tmq\subscribeDb1.py -@REM python3 .\test.py -f 7-tmq\subscribeStb.py -@REM python3 .\test.py -f 7-tmq\subscribeStb0.py -@REM python3 .\test.py -f 7-tmq\subscribeStb1.py -@REM python3 .\test.py -f 7-tmq\subscribeStb2.py -@REM python3 .\test.py -f 7-tmq\subscribeStb3.py -@REM python3 .\test.py -f 7-tmq\subscribeStb4.py -@REM python3 .\test.py -f 7-tmq\db.py +python3 .\test.py -f 7-tmq\basic5.py +python3 .\test.py -f 7-tmq\subscribeDb.py +python3 .\test.py -f 7-tmq\subscribeDb0.py +python3 .\test.py -f 7-tmq\subscribeDb1.py +python3 .\test.py -f 7-tmq\subscribeStb.py +python3 .\test.py -f 7-tmq\subscribeStb0.py +python3 .\test.py -f 7-tmq\subscribeStb1.py +python3 .\test.py -f 7-tmq\subscribeStb2.py +python3 .\test.py -f 7-tmq\subscribeStb3.py +python3 .\test.py -f 7-tmq\subscribeStb4.py +python3 .\test.py -f 7-tmq\db.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 50c3259d68..b9d5227d96 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -100,7 +100,7 @@ python3 ./test.py -f 2-query/statecount.py python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py -python3 ./test.py -f 7-tmq/subscribeDb1.py +# python3 ./test.py -f 7-tmq/subscribeDb1.py python3 ./test.py -f 7-tmq/subscribeStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py python3 ./test.py -f 7-tmq/subscribeStb1.py diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat new file mode 100644 index 0000000000..c8084c3ad5 --- /dev/null +++ b/tests/system-test/simpletest.bat @@ -0,0 +1,103 @@ + +@REM python3 .\test.py -f 0-others\taosShell.py +@REM python3 .\test.py -f 0-others\taosShellError.py +python3 .\test.py -f 0-others\taosShellNetChk.py +python3 .\test.py -f 0-others\telemetry.py +python3 .\test.py -f 0-others\taosdMonitor.py +python3 .\test.py -f 0-others\udfTest.py +python3 .\test.py -f 0-others\udf_create.py +@REM python3 .\test.py -f 0-others\udf_restart_taosd.py +@REM python3 .\test.py -f 0-others\cachelast.py + +@REM python3 .\test.py -f 0-others\user_control.py +@REM python3 .\test.py -f 0-others\fsync.py + +@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py +@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py +@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py +@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py +@REM python3 .\test.py -f 1-insert\alter_stable.py +@REM python3 .\test.py -f 1-insert\alter_table.py +@REM python3 .\test.py -f 2-query\between.py +@REM python3 .\test.py -f 2-query\distinct.py +@REM python3 .\test.py -f 2-query\varchar.py +@REM python3 .\test.py -f 2-query\ltrim.py +@REM python3 .\test.py -f 2-query\rtrim.py +@REM python3 .\test.py -f 2-query\length.py +@REM python3 .\test.py -f 2-query\char_length.py +@REM python3 .\test.py -f 2-query\upper.py +@REM python3 .\test.py -f 2-query\lower.py +@REM python3 .\test.py -f 2-query\join.py +@REM python3 .\test.py -f 2-query\join2.py +@REM python3 .\test.py -f 2-query\cast.py +@REM python3 .\test.py -f 2-query\union.py +@REM python3 .\test.py -f 2-query\union1.py +@REM python3 .\test.py -f 2-query\concat.py +@REM python3 .\test.py -f 2-query\concat2.py +@REM python3 .\test.py -f 2-query\concat_ws.py +@REM python3 .\test.py -f 2-query\concat_ws2.py +@REM python3 .\test.py -f 2-query\check_tsdb.py +@REM python3 .\test.py -f 2-query\spread.py +@REM python3 .\test.py -f 2-query\hyperloglog.py + + +@REM python3 .\test.py -f 2-query\timezone.py +@REM python3 .\test.py -f 2-query\Now.py +@REM python3 .\test.py -f 2-query\Today.py +@REM python3 .\test.py -f 2-query\max.py +@REM python3 .\test.py -f 2-query\min.py +@REM python3 .\test.py -f 2-query\count.py +@REM python3 .\test.py -f 2-query\last.py +@REM python3 .\test.py -f 2-query\first.py +@REM python3 .\test.py -f 2-query\To_iso8601.py +@REM python3 .\test.py -f 2-query\To_unixtimestamp.py +@REM python3 .\test.py -f 2-query\timetruncate.py +@REM python3 .\test.py -f 2-query\diff.py +@REM python3 .\test.py -f 2-query\Timediff.py + +@REM python3 .\test.py -f 2-query\top.py +@REM python3 .\test.py -f 2-query\bottom.py +@REM python3 .\test.py -f 2-query\percentile.py +@REM python3 .\test.py -f 2-query\apercentile.py +@REM python3 .\test.py -f 2-query\abs.py +@REM python3 .\test.py -f 2-query\ceil.py +@REM python3 .\test.py -f 2-query\floor.py +@REM python3 .\test.py -f 2-query\round.py +@REM python3 .\test.py -f 2-query\log.py +@REM python3 .\test.py -f 2-query\pow.py +@REM python3 .\test.py -f 2-query\sqrt.py +@REM python3 .\test.py -f 2-query\sin.py +@REM python3 .\test.py -f 2-query\cos.py +@REM python3 .\test.py -f 2-query\tan.py +@REM python3 .\test.py -f 2-query\arcsin.py +@REM python3 .\test.py -f 2-query\arccos.py +@REM python3 .\test.py -f 2-query\arctan.py +@REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py +@REM # python3 .\test.py -f 2-query\nestedQuery.py +@REM # TD-15983 subquery output duplicate name column. +@REM # Please Xiangyang Guo modify the following script +@REM # python3 .\test.py -f 2-query\nestedQuery_str.py + +@REM python3 .\test.py -f 2-query\avg.py +@REM python3 .\test.py -f 2-query\elapsed.py +@REM python3 .\test.py -f 2-query\csum.py +@REM python3 .\test.py -f 2-query\mavg.py +@REM python3 .\test.py -f 2-query\diff.py +@REM python3 .\test.py -f 2-query\sample.py +@REM python3 .\test.py -f 2-query\function_diff.py +@REM python3 .\test.py -f 2-query\unique.py +@REM python3 .\test.py -f 2-query\stateduration.py +@REM python3 .\test.py -f 2-query\function_stateduration.py +@REM python3 .\test.py -f 2-query\statecount.py + +@REM python3 .\test.py -f 7-tmq\basic5.py +@REM python3 .\test.py -f 7-tmq\subscribeDb.py +@REM python3 .\test.py -f 7-tmq\subscribeDb0.py +@REM python3 .\test.py -f 7-tmq\subscribeDb1.py +@REM python3 .\test.py -f 7-tmq\subscribeStb.py +@REM python3 .\test.py -f 7-tmq\subscribeStb0.py +@REM python3 .\test.py -f 7-tmq\subscribeStb1.py +@REM python3 .\test.py -f 7-tmq\subscribeStb2.py +@REM python3 .\test.py -f 7-tmq\subscribeStb3.py +@REM python3 .\test.py -f 7-tmq\subscribeStb4.py +@REM python3 .\test.py -f 7-tmq\db.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 4c67404e01..076be6563a 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -2,26 +2,43 @@ SETLOCAL EnableDelayedExpansion for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") set /a a=0 +if %1 == full ( + echo Windows Taosd Full Test + set /a exitNum=0 + for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( + for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 ) else ( call :colorEcho 0a "Success" &echo. ) + ) + ) + exit !exitNum! +) echo Windows Taosd Test -for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( +for /F "usebackq tokens=*" %%i in (simpletest.bat) do ( for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( - echo Processing %%i - call :GetTimeSeconds %time% - set time1=!_timeTemp! - echo Start at %time% set /a a+=1 + set timeNow=!time! + echo !a! Processing %%i + call :GetTimeSeconds !timeNow! + set time1=!_timeTemp! + echo Start at !timeNow! call %%i ARG1 > result_!a!.txt 2>error_!a!.txt if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) ) ) @REM echo Linux Taosd Test -@REM for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( +@REM for /F "usebackq tokens=*" %%i in (simpletest.bat) do ( @REM for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( -@REM echo Processing %%i -@REM call :GetTimeSeconds %time% -@REM set time1=!_timeTemp! -@REM echo Start at %time% @REM set /a a+=1 +@REM echo !a! Processing %%i +@REM call :GetTimeSeconds !time! +@REM set time1=!_timeTemp! +@REM echo Start at !time! @REM call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt @REM if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) @REM ) @@ -29,10 +46,11 @@ for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( exit :colorEcho -call :GetTimeSeconds %time% +set timeNow=%time% +call :GetTimeSeconds %timeNow% set time2=%_timeTemp% set /a interTime=%time2% - %time1% -echo End at %time% , cast %interTime%s +echo End at %timeNow% , cast %interTime%s echo off "%~2" findstr /v /a:%1 /R "^$" "%~2" nul @@ -46,16 +64,16 @@ set tt=%tt::= % set index=1 for %%a in (%tt%) do ( if !index! EQU 1 ( - set hh=%%a + set /a hh=%%a )^ else if !index! EQU 2 ( - set mm=%%a + set /a mm=%%a )^ else if !index! EQU 3 ( - set ss=%%a + set /a ss=%%a ) set /a index=index+1 ) -set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% || echo hh:%hh% mm:%mm% ss:%ss% goto :eof \ No newline at end of file diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 4d9e2275f4..2710a00a72 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -128,7 +128,12 @@ void initLogFile() { sprintf(filename,"%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString)); //sprintf(filename, "%s/../log/tmqlog.txt", configDir); - +#ifdef WINDOWS + for (int i = 2; i < sizeof(filename); i++) { + if (filename[i] == ':') filename[i] = '-'; + if (filename[i] == '\0') break; + } +#endif TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", filename); @@ -308,8 +313,10 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) taos_print_row(buf, row, fields, numOfFields); + const char* tbName = tmq_get_table_name(msg); + if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName:"null table"), totalRows, buf); if (0 != g_stConfInfo.saveRowFlag) { saveConsumeContentToTbl(pInfo, buf); } @@ -356,6 +363,8 @@ void build_consumer(SThreadInfo* pInfo) { tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); } + tmq_conf_set(conf, "msg.with.table.name", "true"); + // tmq_conf_set(conf, "client.id", "c-001"); // tmq_conf_set(conf, "enable.auto.commit", "true"); @@ -528,6 +537,12 @@ void* consumeThreadFunc(void* param) { // save consume result into consumeresult table saveConsumeResult(pInfo); + // save rows from per vgroup + taosFprintfFile(g_fp, "======== consumerId: %d, consume rows from per vgroups ========\n", pInfo->consumerId); + for (int32_t i = 0; i < pInfo->numOfVgroups; i++) { + taosFprintfFile(g_fp, "vgroups: %04d, rows: %d\n", pInfo->rowsOfPerVgroups[i][0], pInfo->rowsOfPerVgroups[i][1]); + } + return NULL; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 329842aeb3..8ed0e9ddcf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -886,7 +886,9 @@ void shellGetGrantInfo() { memcpy(expiretime, row[1], fields[1].bytes); memcpy(expired, row[2], fields[2].bytes); - if (strcmp(expiretime, "unlimited") == 0) { + if (strcmp(serverVersion, "community") == 0) { + fprintf(stdout, "Server is Community Edition.\n"); + } else if (strcmp(expiretime, "unlimited") == 0) { fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\n", serverVersion, sinfo); } else { fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\n", serverVersion, sinfo, expiretime); diff --git a/tools/taos-tools b/tools/taos-tools index 717f5aaa5f..1446be9516 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 717f5aaa5f0a1b4d92bb2ae68858fec554fb5eda +Subproject commit 1446be95164e6cda3ff88270f7cfa50d8430503f