diff --git a/src/client/CMakeLists.txt b/2.0/src/client/CMakeLists.txt similarity index 100% rename from src/client/CMakeLists.txt rename to 2.0/src/client/CMakeLists.txt diff --git a/src/client/inc/tscGlobalmerge.h b/2.0/src/client/inc/tscGlobalmerge.h similarity index 100% rename from src/client/inc/tscGlobalmerge.h rename to 2.0/src/client/inc/tscGlobalmerge.h diff --git a/src/client/inc/tscLog.h b/2.0/src/client/inc/tscLog.h similarity index 100% rename from src/client/inc/tscLog.h rename to 2.0/src/client/inc/tscLog.h diff --git a/src/client/inc/tscParseLine.h b/2.0/src/client/inc/tscParseLine.h similarity index 100% rename from src/client/inc/tscParseLine.h rename to 2.0/src/client/inc/tscParseLine.h diff --git a/src/client/inc/tscProfile.h b/2.0/src/client/inc/tscProfile.h similarity index 100% rename from src/client/inc/tscProfile.h rename to 2.0/src/client/inc/tscProfile.h diff --git a/src/client/inc/tscSubquery.h b/2.0/src/client/inc/tscSubquery.h similarity index 100% rename from src/client/inc/tscSubquery.h rename to 2.0/src/client/inc/tscSubquery.h diff --git a/src/client/inc/tscUtil.h b/2.0/src/client/inc/tscUtil.h similarity index 100% rename from src/client/inc/tscUtil.h rename to 2.0/src/client/inc/tscUtil.h diff --git a/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h similarity index 100% rename from src/client/inc/tsclient.h rename to 2.0/src/client/inc/tsclient.h diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h similarity index 100% rename from src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h rename to 2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h diff --git a/src/client/src/TSDBJNIConnector.c b/2.0/src/client/src/TSDBJNIConnector.c similarity index 100% rename from src/client/src/TSDBJNIConnector.c rename to 2.0/src/client/src/TSDBJNIConnector.c diff --git a/src/client/src/taos.def b/2.0/src/client/src/taos.def similarity index 100% rename from src/client/src/taos.def rename to 2.0/src/client/src/taos.def diff --git a/src/client/src/taos.rc.in b/2.0/src/client/src/taos.rc.in similarity index 100% rename from src/client/src/taos.rc.in rename to 2.0/src/client/src/taos.rc.in diff --git a/src/client/src/tscAsync.c b/2.0/src/client/src/tscAsync.c similarity index 100% rename from src/client/src/tscAsync.c rename to 2.0/src/client/src/tscAsync.c diff --git a/src/client/src/tscGlobalmerge.c b/2.0/src/client/src/tscGlobalmerge.c similarity index 100% rename from src/client/src/tscGlobalmerge.c rename to 2.0/src/client/src/tscGlobalmerge.c diff --git a/src/client/src/tscLocal.c b/2.0/src/client/src/tscLocal.c similarity index 100% rename from src/client/src/tscLocal.c rename to 2.0/src/client/src/tscLocal.c diff --git a/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c similarity index 100% rename from src/client/src/tscParseInsert.c rename to 2.0/src/client/src/tscParseInsert.c diff --git a/src/client/src/tscParseLineProtocol.c b/2.0/src/client/src/tscParseLineProtocol.c similarity index 100% rename from src/client/src/tscParseLineProtocol.c rename to 2.0/src/client/src/tscParseLineProtocol.c diff --git a/src/client/src/tscParseOpenTSDB.c b/2.0/src/client/src/tscParseOpenTSDB.c similarity index 100% rename from src/client/src/tscParseOpenTSDB.c rename to 2.0/src/client/src/tscParseOpenTSDB.c diff --git a/src/client/src/tscPrepare.c b/2.0/src/client/src/tscPrepare.c similarity index 100% rename from src/client/src/tscPrepare.c rename to 2.0/src/client/src/tscPrepare.c diff --git a/src/client/src/tscProfile.c b/2.0/src/client/src/tscProfile.c similarity index 100% rename from src/client/src/tscProfile.c rename to 2.0/src/client/src/tscProfile.c diff --git a/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c similarity index 100% rename from src/client/src/tscSQLParser.c rename to 2.0/src/client/src/tscSQLParser.c diff --git a/src/client/src/tscServer.c b/2.0/src/client/src/tscServer.c similarity index 100% rename from src/client/src/tscServer.c rename to 2.0/src/client/src/tscServer.c diff --git a/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c similarity index 100% rename from src/client/src/tscSql.c rename to 2.0/src/client/src/tscSql.c diff --git a/src/client/src/tscStream.c b/2.0/src/client/src/tscStream.c similarity index 100% rename from src/client/src/tscStream.c rename to 2.0/src/client/src/tscStream.c diff --git a/src/client/src/tscSub.c b/2.0/src/client/src/tscSub.c similarity index 100% rename from src/client/src/tscSub.c rename to 2.0/src/client/src/tscSub.c diff --git a/src/client/src/tscSubquery.c b/2.0/src/client/src/tscSubquery.c similarity index 100% rename from src/client/src/tscSubquery.c rename to 2.0/src/client/src/tscSubquery.c diff --git a/src/client/src/tscSystem.c b/2.0/src/client/src/tscSystem.c similarity index 100% rename from src/client/src/tscSystem.c rename to 2.0/src/client/src/tscSystem.c diff --git a/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c similarity index 100% rename from src/client/src/tscUtil.c rename to 2.0/src/client/src/tscUtil.c diff --git a/src/client/tests/CMakeLists.txt b/2.0/src/client/tests/CMakeLists.txt similarity index 100% rename from src/client/tests/CMakeLists.txt rename to 2.0/src/client/tests/CMakeLists.txt diff --git a/src/client/tests/cliTest.cpp b/2.0/src/client/tests/cliTest.cpp similarity index 100% rename from src/client/tests/cliTest.cpp rename to 2.0/src/client/tests/cliTest.cpp diff --git a/src/client/tests/timeParseTest.cpp b/2.0/src/client/tests/timeParseTest.cpp similarity index 100% rename from src/client/tests/timeParseTest.cpp rename to 2.0/src/client/tests/timeParseTest.cpp diff --git a/src/connector/C#/TDengineDriver.cs b/2.0/src/connector/C#/TDengineDriver.cs similarity index 100% rename from src/connector/C#/TDengineDriver.cs rename to 2.0/src/connector/C#/TDengineDriver.cs diff --git a/src/connector/jdbc/CMakeLists.txt b/2.0/src/connector/jdbc/CMakeLists.txt similarity index 100% rename from src/connector/jdbc/CMakeLists.txt rename to 2.0/src/connector/jdbc/CMakeLists.txt diff --git a/src/connector/jdbc/deploy-pom.xml b/2.0/src/connector/jdbc/deploy-pom.xml similarity index 100% rename from src/connector/jdbc/deploy-pom.xml rename to 2.0/src/connector/jdbc/deploy-pom.xml diff --git a/src/connector/jdbc/pom.xml b/2.0/src/connector/jdbc/pom.xml similarity index 100% rename from src/connector/jdbc/pom.xml rename to 2.0/src/connector/jdbc/pom.xml diff --git a/src/connector/jdbc/readme.md b/2.0/src/connector/jdbc/readme.md similarity index 100% rename from src/connector/jdbc/readme.md rename to 2.0/src/connector/jdbc/readme.md diff --git a/src/connector/jdbc/src/main/assembly/assembly-jar.xml b/2.0/src/connector/jdbc/src/main/assembly/assembly-jar.xml similarity index 100% rename from src/connector/jdbc/src/main/assembly/assembly-jar.xml rename to 2.0/src/connector/jdbc/src/main/assembly/assembly-jar.xml diff --git a/src/connector/jdbc/src/main/assembly/assembly.xml b/2.0/src/connector/jdbc/src/main/assembly/assembly.xml similarity index 100% rename from src/connector/jdbc/src/main/assembly/assembly.xml rename to 2.0/src/connector/jdbc/src/main/assembly/assembly.xml diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java diff --git a/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver b/2.0/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver similarity index 100% rename from src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver rename to 2.0/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/2.0/src/connector/nodejs/nodetaos/cinterface.js similarity index 100% rename from src/connector/nodejs/nodetaos/cinterface.js rename to 2.0/src/connector/nodejs/nodetaos/cinterface.js diff --git a/src/connector/nodejs/nodetaos/connection.js b/2.0/src/connector/nodejs/nodetaos/connection.js similarity index 100% rename from src/connector/nodejs/nodetaos/connection.js rename to 2.0/src/connector/nodejs/nodetaos/connection.js diff --git a/src/connector/nodejs/nodetaos/constants.js b/2.0/src/connector/nodejs/nodetaos/constants.js similarity index 100% rename from src/connector/nodejs/nodetaos/constants.js rename to 2.0/src/connector/nodejs/nodetaos/constants.js diff --git a/src/connector/nodejs/nodetaos/cursor.js b/2.0/src/connector/nodejs/nodetaos/cursor.js similarity index 100% rename from src/connector/nodejs/nodetaos/cursor.js rename to 2.0/src/connector/nodejs/nodetaos/cursor.js diff --git a/src/connector/nodejs/nodetaos/error.js b/2.0/src/connector/nodejs/nodetaos/error.js similarity index 100% rename from src/connector/nodejs/nodetaos/error.js rename to 2.0/src/connector/nodejs/nodetaos/error.js diff --git a/src/connector/nodejs/nodetaos/globalfunc.js b/2.0/src/connector/nodejs/nodetaos/globalfunc.js similarity index 100% rename from src/connector/nodejs/nodetaos/globalfunc.js rename to 2.0/src/connector/nodejs/nodetaos/globalfunc.js diff --git a/src/connector/nodejs/nodetaos/taosobjects.js b/2.0/src/connector/nodejs/nodetaos/taosobjects.js similarity index 100% rename from src/connector/nodejs/nodetaos/taosobjects.js rename to 2.0/src/connector/nodejs/nodetaos/taosobjects.js diff --git a/src/connector/nodejs/nodetaos/taosquery.js b/2.0/src/connector/nodejs/nodetaos/taosquery.js similarity index 100% rename from src/connector/nodejs/nodetaos/taosquery.js rename to 2.0/src/connector/nodejs/nodetaos/taosquery.js diff --git a/src/connector/nodejs/nodetaos/taosresult.js b/2.0/src/connector/nodejs/nodetaos/taosresult.js similarity index 100% rename from src/connector/nodejs/nodetaos/taosresult.js rename to 2.0/src/connector/nodejs/nodetaos/taosresult.js diff --git a/src/connector/nodejs/package.json b/2.0/src/connector/nodejs/package.json similarity index 100% rename from src/connector/nodejs/package.json rename to 2.0/src/connector/nodejs/package.json diff --git a/src/connector/nodejs/readme.md b/2.0/src/connector/nodejs/readme.md similarity index 100% rename from src/connector/nodejs/readme.md rename to 2.0/src/connector/nodejs/readme.md diff --git a/src/connector/nodejs/tdengine.js b/2.0/src/connector/nodejs/tdengine.js similarity index 100% rename from src/connector/nodejs/tdengine.js rename to 2.0/src/connector/nodejs/tdengine.js diff --git a/src/connector/nodejs/test/performance.js b/2.0/src/connector/nodejs/test/performance.js similarity index 100% rename from src/connector/nodejs/test/performance.js rename to 2.0/src/connector/nodejs/test/performance.js diff --git a/src/connector/nodejs/test/test.js b/2.0/src/connector/nodejs/test/test.js similarity index 100% rename from src/connector/nodejs/test/test.js rename to 2.0/src/connector/nodejs/test/test.js diff --git a/src/connector/nodejs/test/testMicroseconds.js b/2.0/src/connector/nodejs/test/testMicroseconds.js similarity index 100% rename from src/connector/nodejs/test/testMicroseconds.js rename to 2.0/src/connector/nodejs/test/testMicroseconds.js diff --git a/src/connector/nodejs/test/testNanoseconds.js b/2.0/src/connector/nodejs/test/testNanoseconds.js similarity index 100% rename from src/connector/nodejs/test/testNanoseconds.js rename to 2.0/src/connector/nodejs/test/testNanoseconds.js diff --git a/src/connector/nodejs/test/testSubscribe.js b/2.0/src/connector/nodejs/test/testSubscribe.js similarity index 100% rename from src/connector/nodejs/test/testSubscribe.js rename to 2.0/src/connector/nodejs/test/testSubscribe.js diff --git a/src/connector/nodejs/test/testnchar.js b/2.0/src/connector/nodejs/test/testnchar.js similarity index 100% rename from src/connector/nodejs/test/testnchar.js rename to 2.0/src/connector/nodejs/test/testnchar.js diff --git a/src/connector/odbc/.gitignore b/2.0/src/connector/odbc/.gitignore similarity index 100% rename from src/connector/odbc/.gitignore rename to 2.0/src/connector/odbc/.gitignore diff --git a/src/connector/odbc/CMakeLists.txt b/2.0/src/connector/odbc/CMakeLists.txt similarity index 100% rename from src/connector/odbc/CMakeLists.txt rename to 2.0/src/connector/odbc/CMakeLists.txt diff --git a/src/connector/odbc/README.cn.md b/2.0/src/connector/odbc/README.cn.md similarity index 100% rename from src/connector/odbc/README.cn.md rename to 2.0/src/connector/odbc/README.cn.md diff --git a/src/connector/odbc/README.md b/2.0/src/connector/odbc/README.md similarity index 100% rename from src/connector/odbc/README.md rename to 2.0/src/connector/odbc/README.md diff --git a/src/connector/odbc/examples/CMakeLists.txt b/2.0/src/connector/odbc/examples/CMakeLists.txt similarity index 100% rename from src/connector/odbc/examples/CMakeLists.txt rename to 2.0/src/connector/odbc/examples/CMakeLists.txt diff --git a/src/connector/odbc/examples/c/CMakeLists.txt b/2.0/src/connector/odbc/examples/c/CMakeLists.txt similarity index 100% rename from src/connector/odbc/examples/c/CMakeLists.txt rename to 2.0/src/connector/odbc/examples/c/CMakeLists.txt diff --git a/src/connector/odbc/examples/c/main.c b/2.0/src/connector/odbc/examples/c/main.c similarity index 100% rename from src/connector/odbc/examples/c/main.c rename to 2.0/src/connector/odbc/examples/c/main.c diff --git a/src/connector/odbc/examples/c/main.cpp b/2.0/src/connector/odbc/examples/c/main.cpp similarity index 100% rename from src/connector/odbc/examples/c/main.cpp rename to 2.0/src/connector/odbc/examples/c/main.cpp diff --git a/src/connector/odbc/examples/go/odbc.go b/2.0/src/connector/odbc/examples/go/odbc.go similarity index 100% rename from src/connector/odbc/examples/go/odbc.go rename to 2.0/src/connector/odbc/examples/go/odbc.go diff --git a/src/connector/odbc/examples/js/.gitignore b/2.0/src/connector/odbc/examples/js/.gitignore similarity index 100% rename from src/connector/odbc/examples/js/.gitignore rename to 2.0/src/connector/odbc/examples/js/.gitignore diff --git a/src/connector/odbc/examples/js/odbc.js b/2.0/src/connector/odbc/examples/js/odbc.js similarity index 100% rename from src/connector/odbc/examples/js/odbc.js rename to 2.0/src/connector/odbc/examples/js/odbc.js diff --git a/src/connector/odbc/examples/js/package.json b/2.0/src/connector/odbc/examples/js/package.json similarity index 100% rename from src/connector/odbc/examples/js/package.json rename to 2.0/src/connector/odbc/examples/js/package.json diff --git a/src/connector/odbc/examples/lua/odbc.lua b/2.0/src/connector/odbc/examples/lua/odbc.lua similarity index 100% rename from src/connector/odbc/examples/lua/odbc.lua rename to 2.0/src/connector/odbc/examples/lua/odbc.lua diff --git a/src/connector/odbc/examples/py/odbc.py b/2.0/src/connector/odbc/examples/py/odbc.py similarity index 100% rename from src/connector/odbc/examples/py/odbc.py rename to 2.0/src/connector/odbc/examples/py/odbc.py diff --git a/src/connector/odbc/examples/rust/.gitignore b/2.0/src/connector/odbc/examples/rust/.gitignore similarity index 100% rename from src/connector/odbc/examples/rust/.gitignore rename to 2.0/src/connector/odbc/examples/rust/.gitignore diff --git a/src/connector/odbc/examples/rust/main/Cargo.toml b/2.0/src/connector/odbc/examples/rust/main/Cargo.toml similarity index 100% rename from src/connector/odbc/examples/rust/main/Cargo.toml rename to 2.0/src/connector/odbc/examples/rust/main/Cargo.toml diff --git a/src/connector/odbc/examples/rust/main/src/main.rs b/2.0/src/connector/odbc/examples/rust/main/src/main.rs similarity index 100% rename from src/connector/odbc/examples/rust/main/src/main.rs rename to 2.0/src/connector/odbc/examples/rust/main/src/main.rs diff --git a/src/connector/odbc/samples/create_data.stmts b/2.0/src/connector/odbc/samples/create_data.stmts similarity index 100% rename from src/connector/odbc/samples/create_data.stmts rename to 2.0/src/connector/odbc/samples/create_data.stmts diff --git a/src/connector/odbc/samples/query_data.stmts b/2.0/src/connector/odbc/samples/query_data.stmts similarity index 100% rename from src/connector/odbc/samples/query_data.stmts rename to 2.0/src/connector/odbc/samples/query_data.stmts diff --git a/src/connector/odbc/samples/select.stmts b/2.0/src/connector/odbc/samples/select.stmts similarity index 100% rename from src/connector/odbc/samples/select.stmts rename to 2.0/src/connector/odbc/samples/select.stmts diff --git a/src/connector/odbc/samples/simples.stmts b/2.0/src/connector/odbc/samples/simples.stmts similarity index 100% rename from src/connector/odbc/samples/simples.stmts rename to 2.0/src/connector/odbc/samples/simples.stmts diff --git a/src/connector/odbc/src/CMakeLists.txt b/2.0/src/connector/odbc/src/CMakeLists.txt similarity index 100% rename from src/connector/odbc/src/CMakeLists.txt rename to 2.0/src/connector/odbc/src/CMakeLists.txt diff --git a/src/connector/odbc/src/base.c b/2.0/src/connector/odbc/src/base.c similarity index 100% rename from src/connector/odbc/src/base.c rename to 2.0/src/connector/odbc/src/base.c diff --git a/src/connector/odbc/src/base.h b/2.0/src/connector/odbc/src/base.h similarity index 100% rename from src/connector/odbc/src/base.h rename to 2.0/src/connector/odbc/src/base.h diff --git a/src/connector/odbc/src/base/CMakeLists.txt b/2.0/src/connector/odbc/src/base/CMakeLists.txt similarity index 100% rename from src/connector/odbc/src/base/CMakeLists.txt rename to 2.0/src/connector/odbc/src/base/CMakeLists.txt diff --git a/src/connector/odbc/src/base/conn.c b/2.0/src/connector/odbc/src/base/conn.c similarity index 100% rename from src/connector/odbc/src/base/conn.c rename to 2.0/src/connector/odbc/src/base/conn.c diff --git a/src/connector/odbc/src/base/conn.h b/2.0/src/connector/odbc/src/base/conn.h similarity index 100% rename from src/connector/odbc/src/base/conn.h rename to 2.0/src/connector/odbc/src/base/conn.h diff --git a/src/connector/odbc/src/base/env.c b/2.0/src/connector/odbc/src/base/env.c similarity index 100% rename from src/connector/odbc/src/base/env.c rename to 2.0/src/connector/odbc/src/base/env.c diff --git a/src/connector/odbc/src/base/env.h b/2.0/src/connector/odbc/src/base/env.h similarity index 100% rename from src/connector/odbc/src/base/env.h rename to 2.0/src/connector/odbc/src/base/env.h diff --git a/src/connector/odbc/src/base/err.c b/2.0/src/connector/odbc/src/base/err.c similarity index 100% rename from src/connector/odbc/src/base/err.c rename to 2.0/src/connector/odbc/src/base/err.c diff --git a/src/connector/odbc/src/base/err.h b/2.0/src/connector/odbc/src/base/err.h similarity index 100% rename from src/connector/odbc/src/base/err.h rename to 2.0/src/connector/odbc/src/base/err.h diff --git a/src/connector/odbc/src/base/field.c b/2.0/src/connector/odbc/src/base/field.c similarity index 100% rename from src/connector/odbc/src/base/field.c rename to 2.0/src/connector/odbc/src/base/field.c diff --git a/src/connector/odbc/src/base/field.h b/2.0/src/connector/odbc/src/base/field.h similarity index 100% rename from src/connector/odbc/src/base/field.h rename to 2.0/src/connector/odbc/src/base/field.h diff --git a/src/connector/odbc/src/base/null_conn.c b/2.0/src/connector/odbc/src/base/null_conn.c similarity index 100% rename from src/connector/odbc/src/base/null_conn.c rename to 2.0/src/connector/odbc/src/base/null_conn.c diff --git a/src/connector/odbc/src/base/null_conn.h b/2.0/src/connector/odbc/src/base/null_conn.h similarity index 100% rename from src/connector/odbc/src/base/null_conn.h rename to 2.0/src/connector/odbc/src/base/null_conn.h diff --git a/src/connector/odbc/src/base/param.c b/2.0/src/connector/odbc/src/base/param.c similarity index 100% rename from src/connector/odbc/src/base/param.c rename to 2.0/src/connector/odbc/src/base/param.c diff --git a/src/connector/odbc/src/base/param.h b/2.0/src/connector/odbc/src/base/param.h similarity index 100% rename from src/connector/odbc/src/base/param.h rename to 2.0/src/connector/odbc/src/base/param.h diff --git a/src/connector/odbc/src/base/rs.c b/2.0/src/connector/odbc/src/base/rs.c similarity index 100% rename from src/connector/odbc/src/base/rs.c rename to 2.0/src/connector/odbc/src/base/rs.c diff --git a/src/connector/odbc/src/base/rs.h b/2.0/src/connector/odbc/src/base/rs.h similarity index 100% rename from src/connector/odbc/src/base/rs.h rename to 2.0/src/connector/odbc/src/base/rs.h diff --git a/src/connector/odbc/src/base/stmt.c b/2.0/src/connector/odbc/src/base/stmt.c similarity index 100% rename from src/connector/odbc/src/base/stmt.c rename to 2.0/src/connector/odbc/src/base/stmt.c diff --git a/src/connector/odbc/src/base/stmt.h b/2.0/src/connector/odbc/src/base/stmt.h similarity index 100% rename from src/connector/odbc/src/base/stmt.h rename to 2.0/src/connector/odbc/src/base/stmt.h diff --git a/src/connector/odbc/src/base/tsdb_impl.c b/2.0/src/connector/odbc/src/base/tsdb_impl.c similarity index 100% rename from src/connector/odbc/src/base/tsdb_impl.c rename to 2.0/src/connector/odbc/src/base/tsdb_impl.c diff --git a/src/connector/odbc/src/base/tsdb_impl.h b/2.0/src/connector/odbc/src/base/tsdb_impl.h similarity index 100% rename from src/connector/odbc/src/base/tsdb_impl.h rename to 2.0/src/connector/odbc/src/base/tsdb_impl.h diff --git a/src/connector/odbc/src/col.h b/2.0/src/connector/odbc/src/col.h similarity index 100% rename from src/connector/odbc/src/col.h rename to 2.0/src/connector/odbc/src/col.h diff --git a/src/connector/odbc/src/install.cmd b/2.0/src/connector/odbc/src/install.cmd similarity index 100% rename from src/connector/odbc/src/install.cmd rename to 2.0/src/connector/odbc/src/install.cmd diff --git a/src/connector/odbc/src/install.sh b/2.0/src/connector/odbc/src/install.sh similarity index 100% rename from src/connector/odbc/src/install.sh rename to 2.0/src/connector/odbc/src/install.sh diff --git a/src/connector/odbc/src/todbc.def b/2.0/src/connector/odbc/src/todbc.def similarity index 100% rename from src/connector/odbc/src/todbc.def rename to 2.0/src/connector/odbc/src/todbc.def diff --git a/src/connector/odbc/src/todbc.rc.in b/2.0/src/connector/odbc/src/todbc.rc.in similarity index 100% rename from src/connector/odbc/src/todbc.rc.in rename to 2.0/src/connector/odbc/src/todbc.rc.in diff --git a/src/connector/odbc/src/todbc.rsp b/2.0/src/connector/odbc/src/todbc.rsp similarity index 100% rename from src/connector/odbc/src/todbc.rsp rename to 2.0/src/connector/odbc/src/todbc.rsp diff --git a/src/connector/odbc/src/todbc_buf.c b/2.0/src/connector/odbc/src/todbc_buf.c similarity index 100% rename from src/connector/odbc/src/todbc_buf.c rename to 2.0/src/connector/odbc/src/todbc_buf.c diff --git a/src/connector/odbc/src/todbc_buf.h b/2.0/src/connector/odbc/src/todbc_buf.h similarity index 100% rename from src/connector/odbc/src/todbc_buf.h rename to 2.0/src/connector/odbc/src/todbc_buf.h diff --git a/src/connector/odbc/src/todbc_flex.h b/2.0/src/connector/odbc/src/todbc_flex.h similarity index 100% rename from src/connector/odbc/src/todbc_flex.h rename to 2.0/src/connector/odbc/src/todbc_flex.h diff --git a/src/connector/odbc/src/todbc_hash.c b/2.0/src/connector/odbc/src/todbc_hash.c similarity index 100% rename from src/connector/odbc/src/todbc_hash.c rename to 2.0/src/connector/odbc/src/todbc_hash.c diff --git a/src/connector/odbc/src/todbc_hash.h b/2.0/src/connector/odbc/src/todbc_hash.h similarity index 100% rename from src/connector/odbc/src/todbc_hash.h rename to 2.0/src/connector/odbc/src/todbc_hash.h diff --git a/src/connector/odbc/src/todbc_iconv.c b/2.0/src/connector/odbc/src/todbc_iconv.c similarity index 100% rename from src/connector/odbc/src/todbc_iconv.c rename to 2.0/src/connector/odbc/src/todbc_iconv.c diff --git a/src/connector/odbc/src/todbc_iconv.h b/2.0/src/connector/odbc/src/todbc_iconv.h similarity index 100% rename from src/connector/odbc/src/todbc_iconv.h rename to 2.0/src/connector/odbc/src/todbc_iconv.h diff --git a/src/connector/odbc/src/todbc_list.c b/2.0/src/connector/odbc/src/todbc_list.c similarity index 100% rename from src/connector/odbc/src/todbc_list.c rename to 2.0/src/connector/odbc/src/todbc_list.c diff --git a/src/connector/odbc/src/todbc_list.h b/2.0/src/connector/odbc/src/todbc_list.h similarity index 100% rename from src/connector/odbc/src/todbc_list.h rename to 2.0/src/connector/odbc/src/todbc_list.h diff --git a/src/connector/odbc/src/todbc_log.c b/2.0/src/connector/odbc/src/todbc_log.c similarity index 100% rename from src/connector/odbc/src/todbc_log.c rename to 2.0/src/connector/odbc/src/todbc_log.c diff --git a/src/connector/odbc/src/todbc_log.h b/2.0/src/connector/odbc/src/todbc_log.h similarity index 100% rename from src/connector/odbc/src/todbc_log.h rename to 2.0/src/connector/odbc/src/todbc_log.h diff --git a/src/connector/odbc/src/todbc_scanner.l b/2.0/src/connector/odbc/src/todbc_scanner.l similarity index 100% rename from src/connector/odbc/src/todbc_scanner.l rename to 2.0/src/connector/odbc/src/todbc_scanner.l diff --git a/src/connector/odbc/src/todbc_string.c b/2.0/src/connector/odbc/src/todbc_string.c similarity index 100% rename from src/connector/odbc/src/todbc_string.c rename to 2.0/src/connector/odbc/src/todbc_string.c diff --git a/src/connector/odbc/src/todbc_string.h b/2.0/src/connector/odbc/src/todbc_string.h similarity index 100% rename from src/connector/odbc/src/todbc_string.h rename to 2.0/src/connector/odbc/src/todbc_string.h diff --git a/src/connector/odbc/src/todbc_tls.c b/2.0/src/connector/odbc/src/todbc_tls.c similarity index 100% rename from src/connector/odbc/src/todbc_tls.c rename to 2.0/src/connector/odbc/src/todbc_tls.c diff --git a/src/connector/odbc/src/todbc_tls.h b/2.0/src/connector/odbc/src/todbc_tls.h similarity index 100% rename from src/connector/odbc/src/todbc_tls.h rename to 2.0/src/connector/odbc/src/todbc_tls.h diff --git a/src/connector/odbc/src/todbc_util.c b/2.0/src/connector/odbc/src/todbc_util.c similarity index 100% rename from src/connector/odbc/src/todbc_util.c rename to 2.0/src/connector/odbc/src/todbc_util.c diff --git a/src/connector/odbc/src/todbc_util.h b/2.0/src/connector/odbc/src/todbc_util.h similarity index 100% rename from src/connector/odbc/src/todbc_util.h rename to 2.0/src/connector/odbc/src/todbc_util.h diff --git a/src/connector/odbc/tools/CMakeLists.txt b/2.0/src/connector/odbc/tools/CMakeLists.txt similarity index 100% rename from src/connector/odbc/tools/CMakeLists.txt rename to 2.0/src/connector/odbc/tools/CMakeLists.txt diff --git a/src/connector/odbc/tools/main.c b/2.0/src/connector/odbc/tools/main.c similarity index 100% rename from src/connector/odbc/tools/main.c rename to 2.0/src/connector/odbc/tools/main.c diff --git a/src/connector/odbc/tools/tconv.c b/2.0/src/connector/odbc/tools/tconv.c similarity index 100% rename from src/connector/odbc/tools/tconv.c rename to 2.0/src/connector/odbc/tools/tconv.c diff --git a/src/connector/python/.gitignore b/2.0/src/connector/python/.gitignore similarity index 100% rename from src/connector/python/.gitignore rename to 2.0/src/connector/python/.gitignore diff --git a/src/connector/python/LICENSE b/2.0/src/connector/python/LICENSE similarity index 100% rename from src/connector/python/LICENSE rename to 2.0/src/connector/python/LICENSE diff --git a/src/connector/python/README.md b/2.0/src/connector/python/README.md similarity index 100% rename from src/connector/python/README.md rename to 2.0/src/connector/python/README.md diff --git a/src/connector/python/examples/bind-multi.py b/2.0/src/connector/python/examples/bind-multi.py similarity index 100% rename from src/connector/python/examples/bind-multi.py rename to 2.0/src/connector/python/examples/bind-multi.py diff --git a/src/connector/python/examples/bind-row.py b/2.0/src/connector/python/examples/bind-row.py similarity index 100% rename from src/connector/python/examples/bind-row.py rename to 2.0/src/connector/python/examples/bind-row.py diff --git a/src/connector/python/examples/demo.py b/2.0/src/connector/python/examples/demo.py similarity index 100% rename from src/connector/python/examples/demo.py rename to 2.0/src/connector/python/examples/demo.py diff --git a/src/connector/python/examples/insert-lines.py b/2.0/src/connector/python/examples/insert-lines.py similarity index 100% rename from src/connector/python/examples/insert-lines.py rename to 2.0/src/connector/python/examples/insert-lines.py diff --git a/src/connector/python/examples/pep-249.py b/2.0/src/connector/python/examples/pep-249.py similarity index 100% rename from src/connector/python/examples/pep-249.py rename to 2.0/src/connector/python/examples/pep-249.py diff --git a/src/connector/python/examples/query-async.py b/2.0/src/connector/python/examples/query-async.py similarity index 100% rename from src/connector/python/examples/query-async.py rename to 2.0/src/connector/python/examples/query-async.py diff --git a/src/connector/python/examples/query-objectively.py b/2.0/src/connector/python/examples/query-objectively.py similarity index 100% rename from src/connector/python/examples/query-objectively.py rename to 2.0/src/connector/python/examples/query-objectively.py diff --git a/src/connector/python/examples/subscribe-async.py b/2.0/src/connector/python/examples/subscribe-async.py similarity index 100% rename from src/connector/python/examples/subscribe-async.py rename to 2.0/src/connector/python/examples/subscribe-async.py diff --git a/src/connector/python/examples/subscribe-sync.py b/2.0/src/connector/python/examples/subscribe-sync.py similarity index 100% rename from src/connector/python/examples/subscribe-sync.py rename to 2.0/src/connector/python/examples/subscribe-sync.py diff --git a/src/connector/python/pyproject.toml b/2.0/src/connector/python/pyproject.toml similarity index 100% rename from src/connector/python/pyproject.toml rename to 2.0/src/connector/python/pyproject.toml diff --git a/src/connector/python/setup.py b/2.0/src/connector/python/setup.py similarity index 100% rename from src/connector/python/setup.py rename to 2.0/src/connector/python/setup.py diff --git a/src/connector/python/taos/__init__.py b/2.0/src/connector/python/taos/__init__.py similarity index 100% rename from src/connector/python/taos/__init__.py rename to 2.0/src/connector/python/taos/__init__.py diff --git a/src/connector/python/taos/bind.py b/2.0/src/connector/python/taos/bind.py similarity index 100% rename from src/connector/python/taos/bind.py rename to 2.0/src/connector/python/taos/bind.py diff --git a/src/connector/python/taos/cinterface.py b/2.0/src/connector/python/taos/cinterface.py similarity index 100% rename from src/connector/python/taos/cinterface.py rename to 2.0/src/connector/python/taos/cinterface.py diff --git a/src/connector/python/taos/connection.py b/2.0/src/connector/python/taos/connection.py similarity index 100% rename from src/connector/python/taos/connection.py rename to 2.0/src/connector/python/taos/connection.py diff --git a/src/connector/python/taos/constants.py b/2.0/src/connector/python/taos/constants.py similarity index 100% rename from src/connector/python/taos/constants.py rename to 2.0/src/connector/python/taos/constants.py diff --git a/src/connector/python/taos/cursor.py b/2.0/src/connector/python/taos/cursor.py similarity index 100% rename from src/connector/python/taos/cursor.py rename to 2.0/src/connector/python/taos/cursor.py diff --git a/src/connector/python/taos/error.py b/2.0/src/connector/python/taos/error.py similarity index 100% rename from src/connector/python/taos/error.py rename to 2.0/src/connector/python/taos/error.py diff --git a/src/connector/python/taos/field.py b/2.0/src/connector/python/taos/field.py similarity index 100% rename from src/connector/python/taos/field.py rename to 2.0/src/connector/python/taos/field.py diff --git a/src/connector/python/taos/precision.py b/2.0/src/connector/python/taos/precision.py similarity index 100% rename from src/connector/python/taos/precision.py rename to 2.0/src/connector/python/taos/precision.py diff --git a/src/connector/python/taos/result.py b/2.0/src/connector/python/taos/result.py similarity index 100% rename from src/connector/python/taos/result.py rename to 2.0/src/connector/python/taos/result.py diff --git a/src/connector/python/taos/statement.py b/2.0/src/connector/python/taos/statement.py similarity index 100% rename from src/connector/python/taos/statement.py rename to 2.0/src/connector/python/taos/statement.py diff --git a/src/connector/python/taos/stream.py b/2.0/src/connector/python/taos/stream.py similarity index 100% rename from src/connector/python/taos/stream.py rename to 2.0/src/connector/python/taos/stream.py diff --git a/src/connector/python/taos/subscription.py b/2.0/src/connector/python/taos/subscription.py similarity index 100% rename from src/connector/python/taos/subscription.py rename to 2.0/src/connector/python/taos/subscription.py diff --git a/src/connector/python/taos/timestamp.py b/2.0/src/connector/python/taos/timestamp.py similarity index 100% rename from src/connector/python/taos/timestamp.py rename to 2.0/src/connector/python/taos/timestamp.py diff --git a/src/connector/python/tests/test-td6231.py b/2.0/src/connector/python/tests/test-td6231.py similarity index 100% rename from src/connector/python/tests/test-td6231.py rename to 2.0/src/connector/python/tests/test-td6231.py diff --git a/src/connector/python/tests/test_ctaos.py b/2.0/src/connector/python/tests/test_ctaos.py similarity index 100% rename from src/connector/python/tests/test_ctaos.py rename to 2.0/src/connector/python/tests/test_ctaos.py diff --git a/src/connector/python/tests/test_info.py b/2.0/src/connector/python/tests/test_info.py similarity index 100% rename from src/connector/python/tests/test_info.py rename to 2.0/src/connector/python/tests/test_info.py diff --git a/src/connector/python/tests/test_lines.py b/2.0/src/connector/python/tests/test_lines.py similarity index 100% rename from src/connector/python/tests/test_lines.py rename to 2.0/src/connector/python/tests/test_lines.py diff --git a/src/connector/python/tests/test_query.py b/2.0/src/connector/python/tests/test_query.py similarity index 100% rename from src/connector/python/tests/test_query.py rename to 2.0/src/connector/python/tests/test_query.py diff --git a/src/connector/python/tests/test_query_a.py b/2.0/src/connector/python/tests/test_query_a.py similarity index 100% rename from src/connector/python/tests/test_query_a.py rename to 2.0/src/connector/python/tests/test_query_a.py diff --git a/src/connector/python/tests/test_stmt.py b/2.0/src/connector/python/tests/test_stmt.py similarity index 100% rename from src/connector/python/tests/test_stmt.py rename to 2.0/src/connector/python/tests/test_stmt.py diff --git a/src/connector/python/tests/test_stream.py b/2.0/src/connector/python/tests/test_stream.py similarity index 100% rename from src/connector/python/tests/test_stream.py rename to 2.0/src/connector/python/tests/test_stream.py diff --git a/src/connector/python/tests/test_subscribe.py b/2.0/src/connector/python/tests/test_subscribe.py similarity index 100% rename from src/connector/python/tests/test_subscribe.py rename to 2.0/src/connector/python/tests/test_subscribe.py diff --git a/src/kit/taosdemo/CMakeLists.txt b/2.0/src/kit/taosdemo/CMakeLists.txt similarity index 100% rename from src/kit/taosdemo/CMakeLists.txt rename to 2.0/src/kit/taosdemo/CMakeLists.txt diff --git a/src/kit/taosdemo/async-sub.json b/2.0/src/kit/taosdemo/async-sub.json similarity index 100% rename from src/kit/taosdemo/async-sub.json rename to 2.0/src/kit/taosdemo/async-sub.json diff --git a/src/kit/taosdemo/insert-interlace.json b/2.0/src/kit/taosdemo/insert-interlace.json similarity index 100% rename from src/kit/taosdemo/insert-interlace.json rename to 2.0/src/kit/taosdemo/insert-interlace.json diff --git a/src/kit/taosdemo/insert.json b/2.0/src/kit/taosdemo/insert.json similarity index 100% rename from src/kit/taosdemo/insert.json rename to 2.0/src/kit/taosdemo/insert.json diff --git a/src/kit/taosdemo/query.json b/2.0/src/kit/taosdemo/query.json similarity index 100% rename from src/kit/taosdemo/query.json rename to 2.0/src/kit/taosdemo/query.json diff --git a/src/kit/taosdemo/subscribe.json b/2.0/src/kit/taosdemo/subscribe.json similarity index 100% rename from src/kit/taosdemo/subscribe.json rename to 2.0/src/kit/taosdemo/subscribe.json diff --git a/src/kit/taosdemo/taosdemo.c b/2.0/src/kit/taosdemo/taosdemo.c similarity index 100% rename from src/kit/taosdemo/taosdemo.c rename to 2.0/src/kit/taosdemo/taosdemo.c diff --git a/src/kit/taosdump/CMakeLists.txt b/2.0/src/kit/taosdump/CMakeLists.txt similarity index 100% rename from src/kit/taosdump/CMakeLists.txt rename to 2.0/src/kit/taosdump/CMakeLists.txt diff --git a/src/kit/taosdump/taosdump.c b/2.0/src/kit/taosdump/taosdump.c similarity index 100% rename from src/kit/taosdump/taosdump.c rename to 2.0/src/kit/taosdump/taosdump.c diff --git a/src/kit/taosdump/taosdump.sh b/2.0/src/kit/taosdump/taosdump.sh similarity index 100% rename from src/kit/taosdump/taosdump.sh rename to 2.0/src/kit/taosdump/taosdump.sh diff --git a/src/kit/taospack/CMakeLists.txt b/2.0/src/kit/taospack/CMakeLists.txt similarity index 100% rename from src/kit/taospack/CMakeLists.txt rename to 2.0/src/kit/taospack/CMakeLists.txt diff --git a/src/kit/taospack/taospack.c b/2.0/src/kit/taospack/taospack.c similarity index 100% rename from src/kit/taospack/taospack.c rename to 2.0/src/kit/taospack/taospack.c diff --git a/src/query/CMakeLists.txt b/2.0/src/query/CMakeLists.txt similarity index 100% rename from src/query/CMakeLists.txt rename to 2.0/src/query/CMakeLists.txt diff --git a/src/query/inc/qAggMain.h b/2.0/src/query/inc/qAggMain.h similarity index 100% rename from src/query/inc/qAggMain.h rename to 2.0/src/query/inc/qAggMain.h diff --git a/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h similarity index 100% rename from src/query/inc/qExecutor.h rename to 2.0/src/query/inc/qExecutor.h diff --git a/src/query/inc/qExtbuffer.h b/2.0/src/query/inc/qExtbuffer.h similarity index 100% rename from src/query/inc/qExtbuffer.h rename to 2.0/src/query/inc/qExtbuffer.h diff --git a/src/query/inc/qFill.h b/2.0/src/query/inc/qFill.h similarity index 100% rename from src/query/inc/qFill.h rename to 2.0/src/query/inc/qFill.h diff --git a/src/query/inc/qHistogram.h b/2.0/src/query/inc/qHistogram.h similarity index 100% rename from src/query/inc/qHistogram.h rename to 2.0/src/query/inc/qHistogram.h diff --git a/src/query/inc/qPercentile.h b/2.0/src/query/inc/qPercentile.h similarity index 100% rename from src/query/inc/qPercentile.h rename to 2.0/src/query/inc/qPercentile.h diff --git a/src/query/inc/qPlan.h b/2.0/src/query/inc/qPlan.h similarity index 100% rename from src/query/inc/qPlan.h rename to 2.0/src/query/inc/qPlan.h diff --git a/src/query/inc/qResultbuf.h b/2.0/src/query/inc/qResultbuf.h similarity index 100% rename from src/query/inc/qResultbuf.h rename to 2.0/src/query/inc/qResultbuf.h diff --git a/src/query/inc/qSqlparser.h b/2.0/src/query/inc/qSqlparser.h similarity index 100% rename from src/query/inc/qSqlparser.h rename to 2.0/src/query/inc/qSqlparser.h diff --git a/src/query/inc/qTableMeta.h b/2.0/src/query/inc/qTableMeta.h similarity index 100% rename from src/query/inc/qTableMeta.h rename to 2.0/src/query/inc/qTableMeta.h diff --git a/src/query/inc/qTsbuf.h b/2.0/src/query/inc/qTsbuf.h similarity index 100% rename from src/query/inc/qTsbuf.h rename to 2.0/src/query/inc/qTsbuf.h diff --git a/src/query/inc/qUdf.h b/2.0/src/query/inc/qUdf.h similarity index 100% rename from src/query/inc/qUdf.h rename to 2.0/src/query/inc/qUdf.h diff --git a/src/query/inc/queryLog.h b/2.0/src/query/inc/queryLog.h similarity index 100% rename from src/query/inc/queryLog.h rename to 2.0/src/query/inc/queryLog.h diff --git a/src/query/inc/sql.y b/2.0/src/query/inc/sql.y similarity index 100% rename from src/query/inc/sql.y rename to 2.0/src/query/inc/sql.y diff --git a/src/query/src/qAggMain.c b/2.0/src/query/src/qAggMain.c similarity index 100% rename from src/query/src/qAggMain.c rename to 2.0/src/query/src/qAggMain.c diff --git a/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c similarity index 100% rename from src/query/src/qExecutor.c rename to 2.0/src/query/src/qExecutor.c diff --git a/src/query/src/qExtbuffer.c b/2.0/src/query/src/qExtbuffer.c similarity index 100% rename from src/query/src/qExtbuffer.c rename to 2.0/src/query/src/qExtbuffer.c diff --git a/src/query/src/qFill.c b/2.0/src/query/src/qFill.c similarity index 100% rename from src/query/src/qFill.c rename to 2.0/src/query/src/qFill.c diff --git a/src/query/src/qFilter.c b/2.0/src/query/src/qFilter.c similarity index 100% rename from src/query/src/qFilter.c rename to 2.0/src/query/src/qFilter.c diff --git a/src/query/src/qFilterfunc.c b/2.0/src/query/src/qFilterfunc.c similarity index 100% rename from src/query/src/qFilterfunc.c rename to 2.0/src/query/src/qFilterfunc.c diff --git a/src/query/src/qHistogram.c b/2.0/src/query/src/qHistogram.c similarity index 100% rename from src/query/src/qHistogram.c rename to 2.0/src/query/src/qHistogram.c diff --git a/src/query/src/qPercentile.c b/2.0/src/query/src/qPercentile.c similarity index 100% rename from src/query/src/qPercentile.c rename to 2.0/src/query/src/qPercentile.c diff --git a/src/query/src/qPlan.c b/2.0/src/query/src/qPlan.c similarity index 100% rename from src/query/src/qPlan.c rename to 2.0/src/query/src/qPlan.c diff --git a/src/query/src/qResultbuf.c b/2.0/src/query/src/qResultbuf.c similarity index 100% rename from src/query/src/qResultbuf.c rename to 2.0/src/query/src/qResultbuf.c diff --git a/src/query/src/qSqlParser.c b/2.0/src/query/src/qSqlParser.c similarity index 100% rename from src/query/src/qSqlParser.c rename to 2.0/src/query/src/qSqlParser.c diff --git a/src/query/src/qTableMeta.c b/2.0/src/query/src/qTableMeta.c similarity index 100% rename from src/query/src/qTableMeta.c rename to 2.0/src/query/src/qTableMeta.c diff --git a/src/query/src/qTsbuf.c b/2.0/src/query/src/qTsbuf.c similarity index 100% rename from src/query/src/qTsbuf.c rename to 2.0/src/query/src/qTsbuf.c diff --git a/src/query/src/queryMain.c b/2.0/src/query/src/queryMain.c similarity index 100% rename from src/query/src/queryMain.c rename to 2.0/src/query/src/queryMain.c diff --git a/src/query/src/sql.c b/2.0/src/query/src/sql.c similarity index 100% rename from src/query/src/sql.c rename to 2.0/src/query/src/sql.c diff --git a/src/query/tests/CMakeLists.txt b/2.0/src/query/tests/CMakeLists.txt similarity index 100% rename from src/query/tests/CMakeLists.txt rename to 2.0/src/query/tests/CMakeLists.txt diff --git a/src/query/tests/astTest.cpp b/2.0/src/query/tests/astTest.cpp similarity index 100% rename from src/query/tests/astTest.cpp rename to 2.0/src/query/tests/astTest.cpp diff --git a/src/query/tests/cSortTest.cpp b/2.0/src/query/tests/cSortTest.cpp similarity index 100% rename from src/query/tests/cSortTest.cpp rename to 2.0/src/query/tests/cSortTest.cpp diff --git a/src/query/tests/histogramTest.cpp b/2.0/src/query/tests/histogramTest.cpp similarity index 100% rename from src/query/tests/histogramTest.cpp rename to 2.0/src/query/tests/histogramTest.cpp diff --git a/src/query/tests/patternMatchTest.cpp b/2.0/src/query/tests/patternMatchTest.cpp similarity index 100% rename from src/query/tests/patternMatchTest.cpp rename to 2.0/src/query/tests/patternMatchTest.cpp diff --git a/src/query/tests/percentileTest.cpp b/2.0/src/query/tests/percentileTest.cpp similarity index 100% rename from src/query/tests/percentileTest.cpp rename to 2.0/src/query/tests/percentileTest.cpp diff --git a/src/query/tests/rangeMergeTest.cpp b/2.0/src/query/tests/rangeMergeTest.cpp similarity index 100% rename from src/query/tests/rangeMergeTest.cpp rename to 2.0/src/query/tests/rangeMergeTest.cpp diff --git a/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp similarity index 100% rename from src/query/tests/resultBufferTest.cpp rename to 2.0/src/query/tests/resultBufferTest.cpp diff --git a/src/query/tests/tsBufTest.cpp b/2.0/src/query/tests/tsBufTest.cpp similarity index 100% rename from src/query/tests/tsBufTest.cpp rename to 2.0/src/query/tests/tsBufTest.cpp diff --git a/src/query/tests/unitTest.cpp b/2.0/src/query/tests/unitTest.cpp similarity index 100% rename from src/query/tests/unitTest.cpp rename to 2.0/src/query/tests/unitTest.cpp diff --git a/tests/comparisonTest/cassandra/application.conf b/2.0/tests/comparisonTest/cassandra/application.conf similarity index 100% rename from tests/comparisonTest/cassandra/application.conf rename to 2.0/tests/comparisonTest/cassandra/application.conf diff --git a/tests/comparisonTest/cassandra/cassandratest/pom.xml b/2.0/tests/comparisonTest/cassandra/cassandratest/pom.xml similarity index 100% rename from tests/comparisonTest/cassandra/cassandratest/pom.xml rename to 2.0/tests/comparisonTest/cassandra/cassandratest/pom.xml diff --git a/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java b/2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java similarity index 100% rename from tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java rename to 2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java diff --git a/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java b/2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java similarity index 100% rename from tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java rename to 2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java diff --git a/tests/comparisonTest/cassandra/q1.txt b/2.0/tests/comparisonTest/cassandra/q1.txt similarity index 100% rename from tests/comparisonTest/cassandra/q1.txt rename to 2.0/tests/comparisonTest/cassandra/q1.txt diff --git a/tests/comparisonTest/cassandra/q2.txt b/2.0/tests/comparisonTest/cassandra/q2.txt similarity index 100% rename from tests/comparisonTest/cassandra/q2.txt rename to 2.0/tests/comparisonTest/cassandra/q2.txt diff --git a/tests/comparisonTest/cassandra/q3.txt b/2.0/tests/comparisonTest/cassandra/q3.txt similarity index 100% rename from tests/comparisonTest/cassandra/q3.txt rename to 2.0/tests/comparisonTest/cassandra/q3.txt diff --git a/tests/comparisonTest/cassandra/q4.txt b/2.0/tests/comparisonTest/cassandra/q4.txt similarity index 100% rename from tests/comparisonTest/cassandra/q4.txt rename to 2.0/tests/comparisonTest/cassandra/q4.txt diff --git a/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java b/2.0/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java similarity index 100% rename from tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java rename to 2.0/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java diff --git a/tests/comparisonTest/influxdb/main.go b/2.0/tests/comparisonTest/influxdb/main.go similarity index 100% rename from tests/comparisonTest/influxdb/main.go rename to 2.0/tests/comparisonTest/influxdb/main.go diff --git a/tests/comparisonTest/influxdb/q1.txt b/2.0/tests/comparisonTest/influxdb/q1.txt similarity index 100% rename from tests/comparisonTest/influxdb/q1.txt rename to 2.0/tests/comparisonTest/influxdb/q1.txt diff --git a/tests/comparisonTest/influxdb/q2.txt b/2.0/tests/comparisonTest/influxdb/q2.txt similarity index 100% rename from tests/comparisonTest/influxdb/q2.txt rename to 2.0/tests/comparisonTest/influxdb/q2.txt diff --git a/tests/comparisonTest/influxdb/q3.txt b/2.0/tests/comparisonTest/influxdb/q3.txt similarity index 100% rename from tests/comparisonTest/influxdb/q3.txt rename to 2.0/tests/comparisonTest/influxdb/q3.txt diff --git a/tests/comparisonTest/influxdb/q4.txt b/2.0/tests/comparisonTest/influxdb/q4.txt similarity index 100% rename from tests/comparisonTest/influxdb/q4.txt rename to 2.0/tests/comparisonTest/influxdb/q4.txt diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/pom.xml rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/2.0/tests/comparisonTest/tdengine/CMakeLists.txt similarity index 100% rename from tests/comparisonTest/tdengine/CMakeLists.txt rename to 2.0/tests/comparisonTest/tdengine/CMakeLists.txt diff --git a/tests/comparisonTest/tdengine/makefile b/2.0/tests/comparisonTest/tdengine/makefile similarity index 100% rename from tests/comparisonTest/tdengine/makefile rename to 2.0/tests/comparisonTest/tdengine/makefile diff --git a/tests/comparisonTest/tdengine/q1.txt b/2.0/tests/comparisonTest/tdengine/q1.txt similarity index 100% rename from tests/comparisonTest/tdengine/q1.txt rename to 2.0/tests/comparisonTest/tdengine/q1.txt diff --git a/tests/comparisonTest/tdengine/q2.txt b/2.0/tests/comparisonTest/tdengine/q2.txt similarity index 100% rename from tests/comparisonTest/tdengine/q2.txt rename to 2.0/tests/comparisonTest/tdengine/q2.txt diff --git a/tests/comparisonTest/tdengine/q3.txt b/2.0/tests/comparisonTest/tdengine/q3.txt similarity index 100% rename from tests/comparisonTest/tdengine/q3.txt rename to 2.0/tests/comparisonTest/tdengine/q3.txt diff --git a/tests/comparisonTest/tdengine/q4.txt b/2.0/tests/comparisonTest/tdengine/q4.txt similarity index 100% rename from tests/comparisonTest/tdengine/q4.txt rename to 2.0/tests/comparisonTest/tdengine/q4.txt diff --git a/tests/comparisonTest/tdengine/q5.txt b/2.0/tests/comparisonTest/tdengine/q5.txt similarity index 100% rename from tests/comparisonTest/tdengine/q5.txt rename to 2.0/tests/comparisonTest/tdengine/q5.txt diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/2.0/tests/comparisonTest/tdengine/tdengineTest.c similarity index 100% rename from tests/comparisonTest/tdengine/tdengineTest.c rename to 2.0/tests/comparisonTest/tdengine/tdengineTest.c diff --git a/tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs b/2.0/tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs similarity index 100% rename from tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs rename to 2.0/tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs diff --git a/tests/connectorTest/C#Test/nanosupport/nanotest.cs b/2.0/tests/connectorTest/C#Test/nanosupport/nanotest.cs similarity index 100% rename from tests/connectorTest/C#Test/nanosupport/nanotest.cs rename to 2.0/tests/connectorTest/C#Test/nanosupport/nanotest.cs diff --git a/tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js b/2.0/tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js similarity index 100% rename from tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js rename to 2.0/tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/cinterface.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/cinterface.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/cinterface.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/cinterface.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/connection.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/connection.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/connection.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/connection.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/constants.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/constants.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/constants.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/constants.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/cursor.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/cursor.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/cursor.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/cursor.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/error.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/error.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/error.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/error.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/globalfunc.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/globalfunc.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/globalfunc.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/globalfunc.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/taosobjects.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/taosobjects.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/taosobjects.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/taosobjects.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/taosquery.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/taosquery.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/taosquery.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/taosquery.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/taosresult.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/taosresult.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/taosresult.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/taosresult.js diff --git a/tests/connectorTest/nodejsTest/readme.md b/2.0/tests/connectorTest/nodejsTest/readme.md similarity index 100% rename from tests/connectorTest/nodejsTest/readme.md rename to 2.0/tests/connectorTest/nodejsTest/readme.md diff --git a/tests/connectorTest/nodejsTest/tdengine.js b/2.0/tests/connectorTest/nodejsTest/tdengine.js similarity index 100% rename from tests/connectorTest/nodejsTest/tdengine.js rename to 2.0/tests/connectorTest/nodejsTest/tdengine.js diff --git a/tests/connectorTest/nodejsTest/test/performance.js b/2.0/tests/connectorTest/nodejsTest/test/performance.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/performance.js rename to 2.0/tests/connectorTest/nodejsTest/test/performance.js diff --git a/tests/connectorTest/nodejsTest/test/test.js b/2.0/tests/connectorTest/nodejsTest/test/test.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/test.js rename to 2.0/tests/connectorTest/nodejsTest/test/test.js diff --git a/tests/connectorTest/nodejsTest/test/testMicroseconds.js b/2.0/tests/connectorTest/nodejsTest/test/testMicroseconds.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/testMicroseconds.js rename to 2.0/tests/connectorTest/nodejsTest/test/testMicroseconds.js diff --git a/tests/connectorTest/nodejsTest/test/testNanoseconds.js b/2.0/tests/connectorTest/nodejsTest/test/testNanoseconds.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/testNanoseconds.js rename to 2.0/tests/connectorTest/nodejsTest/test/testNanoseconds.js diff --git a/tests/connectorTest/nodejsTest/test/testSubscribe.js b/2.0/tests/connectorTest/nodejsTest/test/testSubscribe.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/testSubscribe.js rename to 2.0/tests/connectorTest/nodejsTest/test/testSubscribe.js diff --git a/tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py b/2.0/tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py similarity index 100% rename from tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py rename to 2.0/tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py diff --git a/tests/connectorTest/odbcTest/nanosupport/odbc.go b/2.0/tests/connectorTest/odbcTest/nanosupport/odbc.go similarity index 100% rename from tests/connectorTest/odbcTest/nanosupport/odbc.go rename to 2.0/tests/connectorTest/odbcTest/nanosupport/odbc.go diff --git a/tests/connectorTest/odbcTest/nanosupport/odbc.py b/2.0/tests/connectorTest/odbcTest/nanosupport/odbc.py similarity index 100% rename from tests/connectorTest/odbcTest/nanosupport/odbc.py rename to 2.0/tests/connectorTest/odbcTest/nanosupport/odbc.py diff --git a/tests/fuzz/sql-fuzzer.c b/2.0/tests/fuzz/sql-fuzzer.c similarity index 100% rename from tests/fuzz/sql-fuzzer.c rename to 2.0/tests/fuzz/sql-fuzzer.c diff --git a/tests/gotest/batchtest.bat b/2.0/tests/gotest/batchtest.bat similarity index 100% rename from tests/gotest/batchtest.bat rename to 2.0/tests/gotest/batchtest.bat diff --git a/tests/gotest/batchtest.sh b/2.0/tests/gotest/batchtest.sh similarity index 100% rename from tests/gotest/batchtest.sh rename to 2.0/tests/gotest/batchtest.sh diff --git a/tests/gotest/case001/case001.bat b/2.0/tests/gotest/case001/case001.bat similarity index 100% rename from tests/gotest/case001/case001.bat rename to 2.0/tests/gotest/case001/case001.bat diff --git a/tests/gotest/case001/case001.go b/2.0/tests/gotest/case001/case001.go similarity index 100% rename from tests/gotest/case001/case001.go rename to 2.0/tests/gotest/case001/case001.go diff --git a/tests/gotest/case001/case001.sh b/2.0/tests/gotest/case001/case001.sh similarity index 100% rename from tests/gotest/case001/case001.sh rename to 2.0/tests/gotest/case001/case001.sh diff --git a/tests/gotest/case002/case002.bat b/2.0/tests/gotest/case002/case002.bat similarity index 100% rename from tests/gotest/case002/case002.bat rename to 2.0/tests/gotest/case002/case002.bat diff --git a/tests/gotest/case002/case002.go b/2.0/tests/gotest/case002/case002.go similarity index 100% rename from tests/gotest/case002/case002.go rename to 2.0/tests/gotest/case002/case002.go diff --git a/tests/gotest/case002/case002.sh b/2.0/tests/gotest/case002/case002.sh similarity index 100% rename from tests/gotest/case002/case002.sh rename to 2.0/tests/gotest/case002/case002.sh diff --git a/tests/gotest/nanosupport/connector/executor.go b/2.0/tests/gotest/nanosupport/connector/executor.go similarity index 100% rename from tests/gotest/nanosupport/connector/executor.go rename to 2.0/tests/gotest/nanosupport/connector/executor.go diff --git a/tests/gotest/nanosupport/nanoCase.bat b/2.0/tests/gotest/nanosupport/nanoCase.bat similarity index 100% rename from tests/gotest/nanosupport/nanoCase.bat rename to 2.0/tests/gotest/nanosupport/nanoCase.bat diff --git a/tests/gotest/nanosupport/nanoCase.sh b/2.0/tests/gotest/nanosupport/nanoCase.sh similarity index 100% rename from tests/gotest/nanosupport/nanoCase.sh rename to 2.0/tests/gotest/nanosupport/nanoCase.sh diff --git a/tests/gotest/nanosupport/nanosupport.go b/2.0/tests/gotest/nanosupport/nanosupport.go similarity index 100% rename from tests/gotest/nanosupport/nanosupport.go rename to 2.0/tests/gotest/nanosupport/nanosupport.go diff --git a/tests/gotest/test.sh b/2.0/tests/gotest/test.sh similarity index 100% rename from tests/gotest/test.sh rename to 2.0/tests/gotest/test.sh diff --git a/tests/http/restful/http_create_db.c b/2.0/tests/http/restful/http_create_db.c similarity index 100% rename from tests/http/restful/http_create_db.c rename to 2.0/tests/http/restful/http_create_db.c diff --git a/tests/http/restful/http_create_tb.c b/2.0/tests/http/restful/http_create_tb.c similarity index 100% rename from tests/http/restful/http_create_tb.c rename to 2.0/tests/http/restful/http_create_tb.c diff --git a/tests/http/restful/http_drop_db.c b/2.0/tests/http/restful/http_drop_db.c similarity index 100% rename from tests/http/restful/http_drop_db.c rename to 2.0/tests/http/restful/http_drop_db.c diff --git a/tests/http/restful/http_insert_tb.c b/2.0/tests/http/restful/http_insert_tb.c similarity index 100% rename from tests/http/restful/http_insert_tb.c rename to 2.0/tests/http/restful/http_insert_tb.c diff --git a/tests/http/restful/http_query_tb.c b/2.0/tests/http/restful/http_query_tb.c similarity index 100% rename from tests/http/restful/http_query_tb.c rename to 2.0/tests/http/restful/http_query_tb.c diff --git a/tests/http/restful/http_use_db.c b/2.0/tests/http/restful/http_use_db.c similarity index 100% rename from tests/http/restful/http_use_db.c rename to 2.0/tests/http/restful/http_use_db.c diff --git a/tests/nettest/FQDNnettest.sh b/2.0/tests/nettest/FQDNnettest.sh similarity index 100% rename from tests/nettest/FQDNnettest.sh rename to 2.0/tests/nettest/FQDNnettest.sh diff --git a/tests/nettest/TCPUDP.sh b/2.0/tests/nettest/TCPUDP.sh similarity index 100% rename from tests/nettest/TCPUDP.sh rename to 2.0/tests/nettest/TCPUDP.sh diff --git a/tests/nettest/fqdn.sh b/2.0/tests/nettest/fqdn.sh similarity index 100% rename from tests/nettest/fqdn.sh rename to 2.0/tests/nettest/fqdn.sh diff --git a/tests/nettest/tcpudp.sh b/2.0/tests/nettest/tcpudp.sh similarity index 100% rename from tests/nettest/tcpudp.sh rename to 2.0/tests/nettest/tcpudp.sh diff --git a/tests/perftest-scripts/cassandraTestQ1Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ1Loop.sh diff --git a/tests/perftest-scripts/cassandraTestQ2Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ2Loop.sh diff --git a/tests/perftest-scripts/cassandraTestQ3Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ3Loop.sh diff --git a/tests/perftest-scripts/cassandraTestQ4Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ4Loop.sh diff --git a/tests/perftest-scripts/cassandraTestWriteLoop.sh b/2.0/tests/perftest-scripts/cassandraTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/cassandraTestWriteLoop.sh diff --git a/tests/perftest-scripts/coverage_test.sh b/2.0/tests/perftest-scripts/coverage_test.sh similarity index 100% rename from tests/perftest-scripts/coverage_test.sh rename to 2.0/tests/perftest-scripts/coverage_test.sh diff --git a/tests/perftest-scripts/full_test.sh b/2.0/tests/perftest-scripts/full_test.sh similarity index 100% rename from tests/perftest-scripts/full_test.sh rename to 2.0/tests/perftest-scripts/full_test.sh diff --git a/tests/perftest-scripts/influxdbTestQ1Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ1Loop.sh diff --git a/tests/perftest-scripts/influxdbTestQ2Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ2Loop.sh diff --git a/tests/perftest-scripts/influxdbTestQ3Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ3Loop.sh diff --git a/tests/perftest-scripts/influxdbTestQ4Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ4Loop.sh diff --git a/tests/perftest-scripts/influxdbTestWriteLoop.sh b/2.0/tests/perftest-scripts/influxdbTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/influxdbTestWriteLoop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ1Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ1Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ2Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ2Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ3Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ3Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ4Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ4Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestWriteLoop.sh b/2.0/tests/perftest-scripts/opentsdbTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestWriteLoop.sh diff --git a/tests/perftest-scripts/perftest-consistent-inserting-data.sh b/2.0/tests/perftest-scripts/perftest-consistent-inserting-data.sh similarity index 100% rename from tests/perftest-scripts/perftest-consistent-inserting-data.sh rename to 2.0/tests/perftest-scripts/perftest-consistent-inserting-data.sh diff --git a/tests/perftest-scripts/perftest-csv2png.gnuplot b/2.0/tests/perftest-scripts/perftest-csv2png.gnuplot similarity index 100% rename from tests/perftest-scripts/perftest-csv2png.gnuplot rename to 2.0/tests/perftest-scripts/perftest-csv2png.gnuplot diff --git a/tests/perftest-scripts/perftest-daily.sh b/2.0/tests/perftest-scripts/perftest-daily.sh similarity index 100% rename from tests/perftest-scripts/perftest-daily.sh rename to 2.0/tests/perftest-scripts/perftest-daily.sh diff --git a/tests/perftest-scripts/perftest-query.sh b/2.0/tests/perftest-scripts/perftest-query.sh similarity index 100% rename from tests/perftest-scripts/perftest-query.sh rename to 2.0/tests/perftest-scripts/perftest-query.sh diff --git a/tests/perftest-scripts/perftest-taosdemo-compare.sh b/2.0/tests/perftest-scripts/perftest-taosdemo-compare.sh similarity index 100% rename from tests/perftest-scripts/perftest-taosdemo-compare.sh rename to 2.0/tests/perftest-scripts/perftest-taosdemo-compare.sh diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/2.0/tests/perftest-scripts/perftest-taosdemo.sh similarity index 100% rename from tests/perftest-scripts/perftest-taosdemo.sh rename to 2.0/tests/perftest-scripts/perftest-taosdemo.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-13d.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-13d.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-13d.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-1d.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-1d.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-1d.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh diff --git a/tests/perftest-scripts/run-csv.sh b/2.0/tests/perftest-scripts/run-csv.sh similarity index 100% rename from tests/perftest-scripts/run-csv.sh rename to 2.0/tests/perftest-scripts/run-csv.sh diff --git a/tests/perftest-scripts/runInfluxdb-13d-csv.sh b/2.0/tests/perftest-scripts/runInfluxdb-13d-csv.sh similarity index 100% rename from tests/perftest-scripts/runInfluxdb-13d-csv.sh rename to 2.0/tests/perftest-scripts/runInfluxdb-13d-csv.sh diff --git a/tests/perftest-scripts/runInfluxdb.sh b/2.0/tests/perftest-scripts/runInfluxdb.sh similarity index 100% rename from tests/perftest-scripts/runInfluxdb.sh rename to 2.0/tests/perftest-scripts/runInfluxdb.sh diff --git a/tests/perftest-scripts/runTDengine.sh b/2.0/tests/perftest-scripts/runTDengine.sh similarity index 100% rename from tests/perftest-scripts/runTDengine.sh rename to 2.0/tests/perftest-scripts/runTDengine.sh diff --git a/tests/perftest-scripts/runreal-13d-csv.sh b/2.0/tests/perftest-scripts/runreal-13d-csv.sh similarity index 100% rename from tests/perftest-scripts/runreal-13d-csv.sh rename to 2.0/tests/perftest-scripts/runreal-13d-csv.sh diff --git a/tests/perftest-scripts/runreal-1d-csv.sh b/2.0/tests/perftest-scripts/runreal-1d-csv.sh similarity index 100% rename from tests/perftest-scripts/runreal-1d-csv.sh rename to 2.0/tests/perftest-scripts/runreal-1d-csv.sh diff --git a/tests/perftest-scripts/taosdemo-csv2png.gnuplot b/2.0/tests/perftest-scripts/taosdemo-csv2png.gnuplot similarity index 100% rename from tests/perftest-scripts/taosdemo-csv2png.gnuplot rename to 2.0/tests/perftest-scripts/taosdemo-csv2png.gnuplot diff --git a/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot b/2.0/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot similarity index 100% rename from tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot rename to 2.0/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot diff --git a/tests/perftest-scripts/tdengineTestQ1Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ1Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ2Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ2Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ3Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ3Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ4Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ4Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ5Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ5Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ5Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ5Loop.sh diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/2.0/tests/perftest-scripts/tdengineTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/tdengineTestWriteLoop.sh diff --git a/tests/perftest-scripts/tdinternal_coverage_test.sh b/2.0/tests/perftest-scripts/tdinternal_coverage_test.sh similarity index 100% rename from tests/perftest-scripts/tdinternal_coverage_test.sh rename to 2.0/tests/perftest-scripts/tdinternal_coverage_test.sh diff --git a/tests/perftest-scripts/tdinternal_test.sh b/2.0/tests/perftest-scripts/tdinternal_test.sh similarity index 100% rename from tests/perftest-scripts/tdinternal_test.sh rename to 2.0/tests/perftest-scripts/tdinternal_test.sh diff --git a/tests/robust/cluster.sh b/2.0/tests/robust/cluster.sh similarity index 100% rename from tests/robust/cluster.sh rename to 2.0/tests/robust/cluster.sh diff --git a/tests/robust/makefile b/2.0/tests/robust/makefile similarity index 100% rename from tests/robust/makefile rename to 2.0/tests/robust/makefile diff --git a/tests/robust/monitor.sh b/2.0/tests/robust/monitor.sh similarity index 100% rename from tests/robust/monitor.sh rename to 2.0/tests/robust/monitor.sh diff --git a/tests/robust/robust.c b/2.0/tests/robust/robust.c similarity index 100% rename from tests/robust/robust.c rename to 2.0/tests/robust/robust.c diff --git a/tests/stress/.gitignore b/2.0/tests/stress/.gitignore similarity index 100% rename from tests/stress/.gitignore rename to 2.0/tests/stress/.gitignore diff --git a/tests/stress/README.md b/2.0/tests/stress/README.md similarity index 100% rename from tests/stress/README.md rename to 2.0/tests/stress/README.md diff --git a/tests/stress/go.mod b/2.0/tests/stress/go.mod similarity index 100% rename from tests/stress/go.mod rename to 2.0/tests/stress/go.mod diff --git a/tests/stress/main.go b/2.0/tests/stress/main.go similarity index 100% rename from tests/stress/main.go rename to 2.0/tests/stress/main.go diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1f7ec3be1e..01b142e333 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -875,6 +875,7 @@ typedef struct SSubQueryMsg { uint64_t sId; uint64_t queryId; uint64_t taskId; + int8_t taskType; uint32_t contentLen; char msg[]; } SSubQueryMsg; @@ -1520,20 +1521,19 @@ typedef struct SMqSetCVgReq { int32_t vgId; int64_t consumerId; char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_CONSUMER_GROUP_LEN]; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; char* sql; char* logicalPlan; char* physicalPlan; SArray* tasks; // SArray } SMqSetCVgReq; - static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->topicName); - tlen += taosEncodeString(buf, pReq->cGroup); + tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); @@ -1544,7 +1544,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->topicName); - buf = taosDecodeStringTo(buf, pReq->cGroup); + buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); @@ -1552,14 +1552,28 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { return buf; } +typedef struct SMqSetCVgRsp { + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; +} SMqSetCVgRsp; + typedef struct SMqCVConsumeReq { int64_t reqId; int64_t offset; - int64_t clientId; + int64_t consumerId; + int64_t blockingTime; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqCVConsumeReq; +typedef struct SMqConsumeRspBlock { + int32_t bodyLen; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char body[]; +} SMqConsumeRspBlock; + typedef struct SMqCVConsumeRsp { int64_t reqId; int64_t clientId; @@ -1569,7 +1583,7 @@ typedef struct SMqCVConsumeRsp { int32_t skipLogNum; int32_t bodyLen; char topicName[TSDB_TOPIC_FNAME_LEN]; - char body[]; + SMqConsumeRspBlock blocks[]; } SMqCvConsumeRsp; #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index ed7fdea2a8..2ff4d58e0b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -160,7 +160,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) @@ -172,7 +172,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 371cb12405..19438b5dd4 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -48,7 +48,6 @@ typedef struct SOutputData { int8_t compressed; char* pData; bool queryEnd; - int32_t scheduleJobNo; int32_t bufStatus; int64_t useconds; int8_t precision; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 0fc7fd679e..61970ff440 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -84,6 +84,13 @@ void* qGetResultRetrieveMsg(qTaskInfo_t qinfo); */ int32_t qKillTask(qTaskInfo_t qinfo); +/** + * kill the ongoing query asynchronously + * @param qinfo qhandle + * @return + */ +int32_t qAsyncKillTask(qTaskInfo_t qinfo); + /** * return whether query is completed or not * @param qinfo diff --git a/include/libs/function/function.h b/include/libs/function/function.h index bf2937a220..39d8ec3420 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -80,6 +80,13 @@ extern "C" { #define FUNCTION_COV 38 +typedef struct SResultRowEntryInfo { + int8_t hasResult; // result generated, not NULL value + bool initialized; // output buffer has been initialized + bool complete; // query has completed + uint32_t numOfRes; // num of output result in current buffer +} SResultRowEntryInfo; + // determine the real data need to calculated the result enum { BLK_DATA_NO_NEEDED = 0x0, @@ -127,6 +134,8 @@ typedef struct SFunctionFpSet { void (*combine)(struct SQLFunctionCtx *pCtx); } SFunctionFpSet; +extern SFunctionFpSet fpSet[1]; + // sql function runtime context typedef struct SQLFunctionCtx { int32_t size; // number of rows @@ -156,6 +165,7 @@ typedef struct SQLFunctionCtx { SPoint1 start; SPoint1 end; + int32_t columnIndex; SFunctionFpSet* fpSet; } SQLFunctionCtx; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 418c43fab9..ecfa344f85 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -91,6 +91,7 @@ SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); int32_t getNewResColId(); void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); +SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema); #ifdef __cplusplus } diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index f069b68286..ba3539e64e 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -114,10 +114,16 @@ typedef struct SProjectPhyNode { SPhyNode node; } SProjectPhyNode; +typedef struct SDownstreamSource { + SQueryNodeAddr addr; + uint64_t taskId; + uint64_t schedId; +} SDownstreamSource; + typedef struct SExchangePhyNode { SPhyNode node; - uint64_t srcTemplateId; // template id of datasource suplans - SArray *pSrcEndPoints; // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode + uint64_t srcTemplateId; // template id of datasource suplans + SArray *pSrcEndPoints; // SArray, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; typedef enum EAggAlgo { @@ -178,7 +184,7 @@ int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** // @subplan subplan to be schedule // @templateId templateId of a group of datasource subplans of this @subplan // @ep one execution location of this group of datasource subplans -void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); +void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource); int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str); diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 5cc896f1c2..2793e72635 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -28,6 +28,7 @@ OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) OP_ENUM_MACRO(SystemTableScan) +OP_ENUM_MACRO(StreamBlockScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) // OP_ENUM_MACRO(Groupby) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3d5c74d093..02207c4d1b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -38,6 +38,11 @@ enum { JOB_TASK_STATUS_FREEING, }; +enum { + TASK_TYPE_PERSISTENT = 1, + TASK_TYPE_TEMP, +}; + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 51aaa7d903..641b485f4c 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -80,8 +80,8 @@ typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWal typedef struct SWalReadHead { int8_t headVer; - uint8_t msgType; - int8_t reserved[2]; + int16_t msgType; + int8_t reserved; int32_t len; int64_t ingestTs; // not implemented int64_t version; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e93577e620..570c1d8375 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -362,6 +362,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation") #define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error") #define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) //"Job freed") +#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B) //"Task status error") // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b6f38b12ca..1d10869e30 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -179,7 +179,7 @@ extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); -SMsgSendInfo* buildMsgInfoImpl(SRequestObj*); +SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); int taos_init(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 51f267e884..34ab1fb05a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -195,8 +195,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { } asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } else { - SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, pSendMsg); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } tsem_wait(&pRequest->body.rspSem); @@ -257,7 +256,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return pRequest->code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); + SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr)); + + SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 2}; + addr.epAddr[0].port = 7100; + strcpy(addr.epAddr[0].fqdn, "localhost"); + + taosArrayPush(execNode, &addr); + return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob); } typedef struct tmq_t tmq_t; @@ -699,6 +705,8 @@ void* doFetchRow(SRequestObj* pRequest) { assert(pRequest != NULL); SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + SEpSet epSet = {0}; + if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { // All data has returned to App already, no need to try again @@ -706,9 +714,13 @@ void* doFetchRow(SRequestObj* pRequest) { return NULL; } - scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); - setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + return NULL; + } + setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); if (pResultInfo->numOfRows == 0) { return NULL; } @@ -716,8 +728,20 @@ void* doFetchRow(SRequestObj* pRequest) { goto _return; } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; + epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; + SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); + + epSet.numOfEps = pVgroupInfo->numOfEps; + epSet.inUse = pVgroupInfo->inUse; + + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); + epSet.port[i] = pVgroupInfo->epAddr[i].port; + } + } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { pRequest->type = TDMT_VND_SHOW_TABLES; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -735,19 +759,29 @@ void* doFetchRow(SRequestObj* pRequest) { SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + epSet.numOfEps = pVgroupInfo->numOfEps; + epSet.inUse = pVgroupInfo->inUse; + + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); + epSet.port[i] = pVgroupInfo->epAddr[i].port; + } + int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE && pResultInfo->pData != NULL) { + return NULL; } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 497ef1ac95..02e36043dc 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -115,7 +115,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { } } else { assert(pRequest != NULL); - pMsgSendInfo->msgInfo = pRequest->body.requestMsg; + pMsgSendInfo->msgInfo = pRequest->body.requestMsg; } pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 1250402caf..440ef0d728 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -46,17 +47,19 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -TEST(testCase, driverInit_Test) { taos_init(); } - -TEST(testCase, connect_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0); - if (pConn == NULL) { - printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); - } - sleep(3); - taos_close(pConn); +TEST(testCase, driverInit_Test) { + taosInitGlobalCfg(); +// taos_init(); } +//TEST(testCase, connect_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// if (pConn == NULL) { +// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); +// } +// taos_close(pConn); +//} +// //TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -148,30 +151,30 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} // -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// //TEST(testCase, create_dnode_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -254,7 +257,7 @@ TEST(testCase, create_db_Test) { // taos_free_result(pRes); // taos_close(pConn); //} - +// //TEST(testCase, create_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -275,17 +278,14 @@ TEST(testCase, create_db_Test) { // // int32_t numOfFields = taos_num_fields(pRes); // ASSERT_EQ(numOfFields, 0); -// // taos_free_result(pRes); -// taos_close(pConn); -//} // // pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); // if (taos_errno(pRes) != 0) { // printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); // } // -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// pRes = taos_query(pConn, "use abc1"); // taos_free_result(pRes); // pRes = taos_query(pConn, "drop stable `123_$^)`"); // if (taos_errno(pRes) != 0) { @@ -294,26 +294,26 @@ TEST(testCase, create_db_Test) { // // taos_close(pConn); //} - -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); - ASSERT_EQ(taos_errno(pRes), 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); - ASSERT_NE(taos_errno(pRes), 0); - - taos_free_result(pRes); - taos_close(pConn); -} - +// +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); +// ASSERT_EQ(taos_errno(pRes), 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); +// ASSERT_NE(taos_errno(pRes), 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// //TEST(testCase, create_ctable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -324,6 +324,12 @@ TEST(testCase, create_table_Test) { // } // taos_free_result(pRes); // +// pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// // pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); // if (taos_errno(pRes) != 0) { // printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); @@ -332,37 +338,31 @@ TEST(testCase, create_table_Test) { // taos_free_result(pRes); // taos_close(pConn); //} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != nullptr); - -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); // if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); // } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// // taos_free_result(pRes); - - TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +// taos_close(pConn); +//} // //TEST(testCase, show_vgroup_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -456,24 +456,28 @@ TEST(testCase, show_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show tables"); +// TAOS_RES* pRes = taos_query(pConn, "show tables"); // if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// } +// +// pRes = taos_query(pConn, "show abc1.tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); -// ASSERT_TRUE(false); // } // // TAOS_ROW pRow = NULL; // TAOS_FIELD* pFields = taos_fetch_fields(pRes); // int32_t numOfFields = taos_num_fields(pRes); // +// int32_t count = 0; // char str[512] = {0}; +// // while ((pRow = taos_fetch_row(pRes)) != NULL) { // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); +// printf("%d: %s\n", ++count, str); // } // // taos_free_result(pRes); @@ -521,30 +525,30 @@ TEST(testCase, show_stable_Test) { // taosHashCleanup(phash); //} // -TEST(testCase, create_topic_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); - taos_free_result(pRes); - taos_close(pConn); -} - +//TEST(testCase, create_topic_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == nullptr); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// char* sql = "select * from tu"; +// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +// taos_free_result(pRes); +// taos_close(pConn); +//} +// //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -562,7 +566,7 @@ TEST(testCase, create_topic_Test) { // taos_free_result(pRes); // taos_close(pConn); //} - +// //TEST(testCase, projection_query_tables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -613,7 +617,7 @@ TEST(testCase, create_topic_Test) { // taos_free_result(pRes); // taos_close(pConn); //} - +// //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -621,7 +625,7 @@ TEST(testCase, create_topic_Test) { // TAOS_RES* pRes = taos_query(pConn, "use abc1"); // taos_free_result(pRes); // -// pRes = taos_query(pConn, "select ts,k from m1"); +// pRes = taos_query(pConn, "select ts from m1"); // if (taos_errno(pRes) != 0) { // printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); @@ -641,3 +645,37 @@ TEST(testCase, create_topic_Test) { // taos_free_result(pRes); // taos_close(pConn); //} + +TEST(testCase, agg_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use dbv"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tx using st tags(111111111111111)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "select count(*) from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ed88b42d39..772b9bf079 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -230,7 +230,7 @@ typedef struct { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int64_t uid; + uint64_t uid; int32_t cfgVersion; int32_t vgVersion; int8_t hashMethod; // default is 1 diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a6fd2a3c58..5ed801e3ea 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -839,14 +839,12 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 SStbObj *pStb = NULL; int32_t cols = 0; char *pWrite; - char prefix[64] = {0}; + char prefix[TSDB_DB_FNAME_LEN] = {0}; SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) { - return TSDB_CODE_MND_INVALID_DB; - } + if (pDb == NULL) return 0; - tstrncpy(prefix, pShow->db, 64); + tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = (int32_t)strlen(prefix); @@ -855,6 +853,10 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pShow->pIter == NULL) break; if (pStb->dbUid != pDb->uid) { + if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { + mError("Inconsistent table data, name:%s, db:%s, dbUid:%"PRIu64, pStb->name, pDb->name, pDb->uid); + } + sdbRelease(pSdb, pStb); continue; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e786a8972c..4cf7505f74 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -55,8 +55,6 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/ - /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/ mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); return sdbSetTable(pMnode->pSdb, table); @@ -95,14 +93,14 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); pCEp->consumerId = consumerId; taosArrayPush(pSub->assigned, pCEp); - pSub->nextConsumerIdx++; + pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); // build msg SMqSetCVgReq req = { .vgId = pCEp->vgId, .consumerId = consumerId, }; - strcpy(req.cGroup, cgroup); + strcpy(req.cgroup, cgroup); strcpy(req.topicName, topic); strcpy(req.sql, pTopic->sql); strcpy(req.logicalPlan, pTopic->logicalPlan); @@ -170,7 +168,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume .vgId = vgId, .consumerId = pConsumer->consumerId, }; - strcpy(req.cGroup, pConsumer->cgroup); + strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.topicName, pTopic->name); strcpy(req.sql, pTopic->sql); strcpy(req.logicalPlan, pTopic->logicalPlan); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 591367c519..035c05c286 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -92,6 +92,10 @@ static int32_t mndRestoreWal(SMnode *pMnode) { goto WAL_RESTORE_OVER; } + if (walCommit(pWal, sdbVer) != 0) { + goto WAL_RESTORE_OVER; + } + if (walBeginSnapshot(pWal, sdbVer) < 0) { goto WAL_RESTORE_OVER; } @@ -99,7 +103,6 @@ static int32_t mndRestoreWal(SMnode *pMnode) { if (walEndSnapshot(pWal) < 0) { goto WAL_RESTORE_OVER; } - } code = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 93d6d104ff..0bed943b60 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -540,35 +540,41 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t cols = 0; char *pWrite; + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); if (pShow->pIter == NULL) break; - cols = 0; + if (pVgroup->dbUid == pDb->uid) { + cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pVgroup->vgId; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pVgroup->numOfTables; - cols++; - - for (int32_t i = 0; i < pShow->replica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; + *(int32_t *)pWrite = pVgroup->vgId; cols++; - const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + *(int32_t *)pWrite = pVgroup->numOfTables; cols++; + + for (int32_t i = 0; i < pShow->replica; ++i) { + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; + cols++; + + const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + cols++; + } + numOfRows++; } sdbRelease(pSdb, pVgroup); - numOfRows++; } + mndReleaseDb(pMnode, pDb); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 031ed178f0..383073871e 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -42,7 +42,8 @@ typedef struct { SSchema *pSchema; } SSchemaWrapper; -typedef struct SMTbCursor SMTbCursor; +typedef struct SMTbCursor SMTbCursor; +typedef struct SMCtbCursor SMCtbCursor; typedef SVCreateTbReq STbCfg; @@ -64,6 +65,10 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); char * metaTbCursorNext(SMTbCursor *pTbCur); +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); + // Options void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 1a41a02673..ec71777882 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -27,6 +27,7 @@ #include "trpc.h" #include "ttimer.h" #include "tutil.h" +#include "wal.h" #ifdef __cplusplus extern "C" { @@ -161,7 +162,8 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int32_t status; + int8_t status; + int64_t offset; void* dst; SSubQueryMsg* pMsg; } STqTaskItem; @@ -173,34 +175,29 @@ typedef struct STqBuffer { STqTaskItem output[TQ_BUFFER_SIZE]; } STqBuffer; -typedef struct STqClientHandle { - int64_t clientId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_TOPIC_FNAME_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - int64_t committedOffset; - int64_t currentOffset; - STqBuffer buffer; -} STqClientHandle; +typedef struct STqTopicHandle { + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_TOPIC_FNAME_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + int64_t committedOffset; + int64_t currentOffset; + STqBuffer buffer; + SWalReadHandle* pReadhandle; +} STqTopicHandle; + +typedef struct STqConsumerHandle { + int64_t consumerId; + int64_t epoch; + SArray* topics; // SArray +} STqConsumerHandle; typedef struct STqQueryMsg { STqMsgItem* item; struct STqQueryMsg* next; } STqQueryMsg; -typedef struct STqLogHandle { - void* logHandle; - void* (*openLogReader)(void* logHandle); - void (*closeLogReader)(void* logReader); - int32_t (*logRead)(void* logReader, void** data, int64_t ver); - - int64_t (*logGetFirstVer)(void* logHandle); - int64_t (*logGetSnapshotVer)(void* logHandle); - int64_t (*logGetLastVer)(void* logHandle); -} STqLogHandle; - typedef struct STqCfg { // TODO } STqCfg; @@ -298,9 +295,9 @@ typedef struct STQ { // the handle of meta kvstore char* path; STqCfg* tqConfig; - STqLogHandle* tqLogHandle; STqMemRef tqMemRef; STqMetaStore* tqMeta; + SWal* pWal; } STQ; typedef struct STqMgmt { @@ -315,31 +312,27 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); +#if 0 +int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); int tqSetCursor(STQ*, STqSetCurReq* pMsg); int tqBufferSetOffset(STqTopic*, int64_t offset); - STqTopic* tqFindTopic(STqGroup*, int64_t topicId); - STqGroup* tqGetGroup(STQ*, int64_t clientId); - STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); int tqRegisterContext(STqGroup*, void* ahandle); int tqSendLaunchQuery(STqMsgItem*, int64_t offset); +#endif -int tqSerializeGroup(const STqGroup*, STqSerializedHead**); - -const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**); - -static int tqQueryExecuting(int32_t status) { return status; } +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); +int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq); typedef struct STqReadHandle { int64_t ver; @@ -348,16 +341,15 @@ typedef struct STqReadHandle { SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; SMeta* pMeta; + SArray* pColumnIdList; } STqReadHandle; -typedef struct SSubmitBlkScanInfo { -} SSubmitBlkScanInfo; - -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg* pMsg); +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList); +void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle* pHandle); int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); // return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList); +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index cfd8e501f6..362ad9b2d8 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -93,6 +93,7 @@ int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); typedef void* tsdbReadHandleT; + /** * Get the data block iterator, starting from position according to the query condition * @@ -124,6 +125,24 @@ tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGro bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); +/** + * + * @param tsdb + * @param uid + * @param skey + * @param pTagCond + * @param len + * @param tagNameRelType + * @param tbnameCond + * @param pGroupInfo + * @param pColIndex + * @param numOfCols + * @param reqId + * @return + */ +int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, + int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId); /** * get num of rows in mem table * diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 107f5d5103..b4e1f57384 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -43,6 +43,9 @@ extern int32_t tqDebugFlag; // delete persistent storage for meta info // int tqDropTCGroup(STQ*, const char* topic, int cgId); +int tqSerializeGroup(const STqGroup*, STqSerializedHead**); +const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); +static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index ae13d9fddc..d8ee48ff23 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -374,22 +374,27 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey DBT * pDbt; if (pTbCfg->type == META_CHILD_TABLE) { - pDbt = calloc(2, sizeof(DBT)); + // pDbt = calloc(2, sizeof(DBT)); - // First key is suid - pDbt[0].data = &(pTbCfg->ctbCfg.suid); - pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); + // // First key is suid + // pDbt[0].data = &(pTbCfg->ctbCfg.suid); + // pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); - // Second key is the first tag - void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - pDbt[1].data = pTagVal; - pDbt[1].size = sizeof(int32_t); + // // Second key is the first tag + // void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); + // pDbt[1].data = pTagVal; + // pDbt[1].size = sizeof(int32_t); // Set index key memset(pSKey, 0, sizeof(*pSKey)); +#if 0 pSKey->flags = DB_DBT_MULTIPLE | DB_DBT_APPMALLOC; pSKey->data = pDbt; pSKey->size = 2; +#else + pSKey->data = &(pTbCfg->ctbCfg.suid); + pSKey->size = sizeof(pTbCfg->ctbCfg.suid); +#endif return 0; } else { @@ -621,4 +626,61 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { tdDestroyTSchemaBuilder(&sb); return pTSchema; +} + +struct SMCtbCursor { + DBC * pCur; + tb_uid_t suid; +}; + +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { + SMCtbCursor *pCtbCur = NULL; + SMetaDB * pDB = pMeta->pDB; + int ret; + + pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); + if (pCtbCur == NULL) { + return NULL; + } + + pCtbCur->suid = uid; + ret = pDB->pCtbIdx->cursor(pDB->pCtbIdx, NULL, &(pCtbCur->pCur), 0); + if (ret != 0) { + free(pCtbCur); + return NULL; + } + + return pCtbCur; +} + +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { + if (pCtbCur) { + if (pCtbCur->pCur) { + pCtbCur->pCur->close(pCtbCur->pCur); + } + + free(pCtbCur); + } +} + +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { + DBT skey = {0}; + DBT pkey = {0}; + DBT pval = {0}; + void * pBuf; + STbCfg tbCfg; + + // Set key + skey.data = &(pCtbCur->suid); + skey.size = sizeof(pCtbCur->suid); + + if (pCtbCur->pCur->pget(pCtbCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) { + tb_uid_t id = *(tb_uid_t *)pkey.data; + assert(id != 0); + return id; + // metaDecodeTbInfo(pBuf, &tbCfg); + // return tbCfg.; + } else { + return 0; + } } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a5be0ec29a..ead856a06b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -37,7 +37,7 @@ const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); - if(old == 1) return 0; + if (old == 1) return 0; tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); return 0; @@ -45,12 +45,12 @@ int tqInit() { void tqCleanUp() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0); - if(old == 0) return; + if (old == 0) return; taosTmrStop(tqMgmt.timer); taosTmrCleanUp(tqMgmt.timer); } -STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -58,7 +58,6 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemA } pTq->path = strdup(path); pTq->tqConfig = tqConfig; - pTq->tqLogHandle = tqLogHandle; #if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); @@ -150,7 +149,7 @@ int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup memset(pGroup, 0, sizeof(STqGroup)); pGroup->topicList = tdListNew(sizeof(STqTopic)); - if(pGroup->topicList == NULL) { + if (pGroup->topicList == NULL) { free(pGroup); return -1; } @@ -190,7 +189,7 @@ static int tqFetch(STqGroup* pGroup, STqConsumeRsp** pRsp) { int totSize = 0; int numOfMsgs = 0; // TODO: make it a macro - int sizeLimit = 4 * 1024; + int sizeLimit = 4 * 1024; void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + sizeLimit); if (ptr == NULL) { @@ -329,9 +328,9 @@ int tqProcessCMsg(STQ* pTq, STqConsumeReq* pMsg, STqRspHandle* pRsp) { } int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { - STqConsumeReq *pMsg = pReq->pCont; - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); + STqConsumeReq* pMsg = pReq->pCont; + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); if (pGroup == NULL) { terrno = TSDB_CODE_TQ_GROUP_NOT_SET; return -1; @@ -343,9 +342,8 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { int numOfMsgs = 0; int sizeLimit = 4096; - - STqConsumeRsp *pCsmRsp = (*pRsp)->pCont; - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); + STqConsumeRsp* pCsmRsp = (*pRsp)->pCont; + void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); if (ptr == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -356,16 +354,16 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { tdListInitIter(topicList, &iter, TD_LIST_FORWARD); STqMsgContent* buffer = NULL; - SArray* pArray = taosArrayInit(0, sizeof(void*)); + SArray* pArray = taosArrayInit(0, sizeof(void*)); - SListNode *pn; - while((pn = tdListNext(&iter)) != NULL) { - STqTopic* pTopic = *(STqTopic**)pn->data; - int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; + SListNode* pn; + while ((pn = tdListNext(&iter)) != NULL) { + STqTopic* pTopic = *(STqTopic**)pn->data; + int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; STqMsgItem* pItem = &pTopic->buffer[idx]; if (pItem->content != NULL && pItem->offset == pTopic->floatingCursor) { - if(pItem->status == TQ_ITEM_READY) { - //if has data + if (pItem->status == TQ_ITEM_READY) { + // if has data totSize += pTopic->buffer[idx].size; if (totSize > sizeLimit) { void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + totSize); @@ -388,13 +386,13 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { if (totSize > sizeLimit) { break; } - } else if(pItem->status == TQ_ITEM_PROCESS) { - //if not have data but in process + } else if (pItem->status == TQ_ITEM_PROCESS) { + // if not have data but in process - } else if(pItem->status == TQ_ITEM_EMPTY){ - //if not have data and not in process + } else if (pItem->status == TQ_ITEM_EMPTY) { + // if not have data and not in process int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_EMPTY, TQ_ITEM_PROCESS); - if(old != TQ_ITEM_EMPTY) { + if (old != TQ_ITEM_EMPTY) { continue; } pItem->offset = pTopic->floatingCursor; @@ -416,22 +414,22 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { } // fetched a num of msgs, rpc response - for(int i = 0; i < pArray->size; i++) { + for (int i = 0; i < pArray->size; i++) { STqMsgItem* pItem = taosArrayGet(pArray, i); - //read from wal + // read from wal void* raw = NULL; /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ - int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset); - if(code < 0) { - //TODO: error - } - //get msgType - //if submitblk + /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ + /*if (code < 0) {*/ + // TODO: error + /*}*/ + // get msgType + // if submitblk pItem->executor->assign(pItem->executor->runtimeEnv, raw); SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); pItem->content = content; - //if other type, send just put into buffer + // if other type, send just put into buffer /*pItem->content = raw;*/ int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); @@ -608,69 +606,163 @@ int tqItemSSize() { return 0; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg *pMsg) { +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { + SMqCVConsumeReq* pReq = pMsg->pCont; + int64_t reqId = pReq->reqId; + int64_t consumerId = pReq->consumerId; + int64_t offset = pReq->offset; + int64_t blockingTime = pReq->blockingTime; + + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); + int sz = taosArrayGetSize(pConsumer->topics); + + for (int i = 0 ; i < sz; i++) { + STqTopicHandle *pHandle = taosArrayGet(pConsumer->topics, i); + + int8_t pos = offset % TQ_BUFFER_SIZE; + int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); + if (old == 1) { + // do nothing + continue; + } + if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { + // TODO + } + SWalHead* pHead = pHandle->pReadhandle->pHead; + while (pHead->head.msgType != TDMT_VND_SUBMIT) { + // read until find TDMT_VND_SUBMIT + } + SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; + + SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; + + // TODO: launch query and get output data + void* outputData; + pHandle->buffer.output[pos].dst = outputData; + if (pHandle->buffer.firstOffset == -1 + || pReq->offset < pHandle->buffer.firstOffset) { + pHandle->buffer.firstOffset = pReq->offset; + } + if (pHandle->buffer.lastOffset == -1 + || pReq->offset > pHandle->buffer.lastOffset) { + pHandle->buffer.lastOffset = pReq->offset; + } + atomic_store_8(&pHandle->buffer.output[pos].status, 1); + + // put output into rsp + } + + // launch query + // get result + SMqCvConsumeRsp* pRsp; + return 0; +} + +int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { + STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); + if (pConsumer == NULL) { + return -1; + } + + STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); + if (pTopic == NULL) { + free(pConsumer); + return -1; + } + strcpy(pTopic->topicName, pReq->topicName); + strcpy(pTopic->cgroup, pReq->cgroup); + strcpy(pTopic->sql, pReq->sql); + strcpy(pTopic->logicalPlan, pReq->logicalPlan); + strcpy(pTopic->physicalPlan, pReq->physicalPlan); + SArray *pArray; + //TODO: deserialize to SQueryDag + SQueryDag *pDag; + // convert to task + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + // TODO: handle error + } + ASSERT(taosArrayGetSize(pArray) == 0); + STaskInfo *pInfo = taosArrayGet(pArray, 0); + SArray* pTasks; + schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); + pTopic->buffer.firstOffset = -1; + pTopic->buffer.lastOffset = -1; + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { + SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); + pTopic->buffer.output[i].pMsg = pMsg; + pTopic->buffer.output[i].status = 0; + } + pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); + // write mq meta + return 0; +} + +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) { STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; } pReadHandle->pMeta = pMeta; - pReadHandle->pMsg = pMsg; - tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); + pReadHandle->pMsg = NULL; pReadHandle->ver = -1; + pReadHandle->pColumnIdList = pColumnIdList; return NULL; } +void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { + pReadHandle->pMsg = pMsg; + tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); + pReadHandle->ver = ver; + memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); +} + bool tqNextDataBlock(STqReadHandle* pHandle) { - if(tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } return true; } int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - SMemRow row; - int32_t sversion = pHandle->pBlock->sversion; + SMemRow row; + int32_t sversion = pHandle->pBlock->sversion; SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false); pBlockInfo->numOfCols = pSchema->nCols; pBlockInfo->rows = pHandle->pBlock->numOfRows; pBlockInfo->uid = pHandle->pBlock->uid; - //TODO: filter out unused column + // TODO: filter out unused column return 0; } -SArray *tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) { - int32_t sversion = pHandle->pBlock->sversion; +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { + int32_t sversion = pHandle->pBlock->sversion; SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true); - STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); - SArray *pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); + STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); + SArray* pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); if (pArray == NULL) { return NULL; } SColumnInfoData colInfo; - int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes; + int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes; colInfo.pData = malloc(sz); if (colInfo.pData == NULL) { return NULL; } - for (int i = 0; i < pTschema->numOfCols; i++) { - //TODO: filter out unused column - taosArrayPush(pColumnIdList, &(schemaColAt(pTschema, i)->colId)); - } - SMemRow row; int32_t kvIdx; while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { for (int i = 0; i < pTschema->numOfCols && kvIdx < pTschema->numOfCols; i++) { - //TODO: filter out unused column - STColumn *pCol = schemaColAt(pTschema, i); + // TODO: filter out unused column + STColumn* pCol = schemaColAt(pTschema, i); void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - //TODO: handle varlen + // TODO: handle varlen memcpy(POINTER_SHIFT(colInfo.pData, pCol->offset), val, pCol->bytes); } } taosArrayPush(pArray, &colInfo); return pArray; } -/*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t status) {*/ - /*return 0;*/ +/*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t + * status) {*/ +/*return 0;*/ /*}*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d07a5ffc77..15748118d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -152,7 +152,7 @@ typedef struct STsdbReadHandle { typedef struct STableGroupSupporter { int32_t numOfCols; SColIndex* pCols; - STSchema* pTagSchema; + SSchema* pTagSchema; } STableGroupSupporter; static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); @@ -466,7 +466,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, return (tsdbReadHandleT)pReadHandle; _end: -// tsdbCleanupQueryHandle(pTsdbReadHandle); + tsdbCleanupQueryHandle(pReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } @@ -2630,18 +2630,20 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int return numOfRows; } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); +static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { + SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid); - STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode); + while (1) { + tb_uid_t id = metaCtbCursorNext(pCur); + if (id == 0) { + break; + } - STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = NULL, .lastKey = TSKEY_INITIAL_VAL, uid = id}; taosArrayPush(list, &info); } - tSkipListDestroyIter(iter); + metaCloseCtbCurosr(pCur); return TSDB_CODE_SUCCESS; } @@ -3553,7 +3555,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } -SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { +SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3564,25 +3566,18 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + SArray* sa = taosArrayDup(pTableList); if (sa == NULL) { taosArrayDestroy(pTableGroup); return NULL; } - for(int32_t i = 0; i < size; ++i) { - STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); - - STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; - taosArrayPush(sa, &info); - } - taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { STableGroupSupporter sup = {0}; sup.numOfCols = numOfOrderCols; - sup.pTagSchema = pTagSchema; + sup.pTagSchema = pTagSchema->pSchema; sup.pCols = pCols; // taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); @@ -3710,12 +3705,11 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - STSchema* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - assert(false); - int32_t ret = 0;//getAllTableList(pTable, res); + int32_t ret = getAllTableList(tsdb->pMeta, uid, res); if (ret != TSDB_CODE_SUCCESS) { goto _error; } @@ -3854,7 +3848,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGro return TSDB_CODE_SUCCESS; } - +#endif static void* doFreeColumnInfoData(SArray* pColumnInfoData) { if (pColumnInfoData == NULL) { return NULL; @@ -3883,6 +3877,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { return NULL; } + void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; if (pTsdbReadHandle == NULL) { @@ -3921,6 +3916,7 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { tfree(pTsdbReadHandle); } +#if 0 void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { assert(pGroupList != NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 76b7ccf0d9..c4bbd93eda 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -117,14 +117,6 @@ static int vnodeOpenImpl(SVnode *pVnode) { return -1; } - // TODO: Open TQ - sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, vBufPoolGetMAF(pVnode)); - if (pVnode->pTq == NULL) { - // TODO: handle error - return -1; - } - // Open WAL sprintf(dir, "%s/wal", pVnode->path); pVnode->pWal = walOpen(dir, &(pVnode->config.walCfg)); @@ -133,6 +125,14 @@ static int vnodeOpenImpl(SVnode *pVnode) { return -1; } + // Open TQ + sprintf(dir, "%s/tq", pVnode->path); + pVnode->pTq = tqOpen(dir, pVnode->pWal, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + if (pVnode->pTq == NULL) { + // TODO: handle error + return -1; + } + // Open Query if (vnodeQueryOpen(pVnode)) { return -1; @@ -151,4 +151,4 @@ static void vnodeCloseImpl(SVnode *pVnode) { tqClose(pVnode->pTq); walClose(pVnode->pWal); } -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 729d64f8b3..dd1e5ba9ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -29,8 +29,6 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); case TDMT_VND_QUERY_CONTINUE: return qWorkerProcessCQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); - case TDMT_VND_SCHEDULE_DATA_SINK: - return qWorkerProcessDataSinkMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -58,7 +56,7 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg, pRsp); case TDMT_VND_CONSUME: - return 0; + return tqProcessConsumeReq(pVnode->pTq, pMsg, pRsp); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -166,16 +164,26 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { char * name = NULL; int32_t totalLen = 0; + int32_t numOfTables = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { - taosArrayPush(pArray, &name); - totalLen += strlen(name); + if (numOfTables < 1000) { // TODO: temp get tables of vnode, and should del when show tables commad ok. + taosArrayPush(pArray, &name); + totalLen += strlen(name); + } + numOfTables++; + } + + // TODO: temp debug, and should del when show tables command ok + vError("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); + if (numOfTables > 1000) { + numOfTables = 1000; } metaCloseTbCursor(pCur); int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); + //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; @@ -203,5 +211,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); + taosArrayDestroy(pArray); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 2f3a4d5409..d1b529f7fb 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tq.h" +#include "vnd.h" int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + void *ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -110,39 +110,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_SET_CONN: { - char* reqStr = ptr; SMqSetCVgReq req; - tDecodeSMqSetCVgReq(reqStr, &req); - STqClientHandle* pHandle = calloc(sizeof(STqClientHandle), 1); - if (pHandle == NULL) { - // TODO: handle error + tDecodeSMqSetCVgReq(ptr, &req); + if (tqProcessSetConnReq(pVnode->pTq, &req) < 0) { } - strcpy(pHandle->topicName, req.topicName); - strcpy(pHandle->cGroup, req.cGroup); - strcpy(pHandle->sql, req.sql); - strcpy(pHandle->logicalPlan, req.logicalPlan); - strcpy(pHandle->physicalPlan, req.physicalPlan); - SArray *pArray; - //TODO: deserialize to SQueryDag - SQueryDag *pDag; - // convert to task - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - // TODO: handle error - } - ASSERT(taosArrayGetSize(pArray) == 0); - STaskInfo *pInfo = taosArrayGet(pArray, 0); - SArray* pTasks; - schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); - pHandle->buffer.firstOffset = -1; - pHandle->buffer.lastOffset = -1; - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); - pHandle->buffer.output[i].pMsg = pMsg; - pHandle->buffer.output[i].status = 0; - } - // write mq meta - } - break; + } break; default: ASSERT(0); break; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 79766b22ac..f552afcfbf 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -94,7 +94,7 @@ struct SUdfInfo; int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); -size_t getResultRowSize(struct STaskRuntimeEnv* pRuntimeEnv); +size_t getResultRowSize(SArray* pExprInfo); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index adb305ab09..2fe3392b25 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -375,14 +375,17 @@ typedef struct STaskParam { } STaskParam; typedef struct SExchangeInfo { - int32_t numOfSources; - SEpSet *pEpset; - int32_t bytes; // total load bytes from remote + SArray *pSources; + uint64_t bytes; // total load bytes from remote + tsem_t ready; + void *pTransporter; + SRetrieveTableRsp *pRsp; + SSDataBlock *pResult; } SExchangeInfo; typedef struct STableScanInfo { void *pTsdbReadHandle; - int32_t numOfBlocks; + int32_t numOfBlocks; // extract basic running information. int32_t numOfSkipped; int32_t numOfBlockStatis; int64_t numOfRows; @@ -411,6 +414,14 @@ typedef struct STagScanInfo { int32_t curPos; } STagScanInfo; +typedef struct SStreamBlockScanInfo { + SSDataBlock *pRes; // result SSDataBlock + SColumnInfo *pCols; // the output column info + uint64_t numOfRows; // total scanned rows + uint64_t numOfExec; // execution times + void *readerHandle;// stream block reader handle +} SStreamBlockScanInfo; + typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; int32_t *rowCellInfoOffset; // offset value for each row result cell info @@ -421,8 +432,15 @@ typedef struct SOptrBasicInfo { typedef struct SOptrBasicInfo STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - uint32_t seed; + SOptrBasicInfo binfo; + uint32_t seed; + SDiskbasedResultBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + STableQueryInfo *current; } SAggOperatorInfo; typedef struct SProjectOperatorInfo { @@ -543,15 +561,14 @@ typedef struct SOrderOperatorInfo { SSDataBlock *pDataBlock; } SOrderOperatorInfo; -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); - -SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t numOfSources, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index edba4fc97d..f9e61f91de 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -196,7 +196,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->bufStatus = updateStatus(pDispatcher); pthread_mutex_lock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; - pOutput->scheduleJobNo = 0; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->schema.precision; pthread_mutex_unlock(&pDispatcher->mutex); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5253add5af..78093ce080 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -168,16 +168,14 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_ } // TODO refactor: use macro -struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { +SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { assert(index >= 0 && offset != NULL); -// return (SResultRowEntryInfo*)((char*) pRow->pCellInfo + offset[index]); -return NULL; + return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]); } -size_t getResultRowSize(STaskRuntimeEnv* pRuntimeEnv) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - return 0; -// return (pQueryAttr->numOfOutput * sizeof(SResultRowEntryInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); +size_t getResultRowSize(SArray* pExprInfo) { + size_t numOfOutput = taosArrayGetSize(pExprInfo); + return (numOfOutput * sizeof(SResultRowEntryInfo)) + /*pQueryAttr->interBufSize +*/ sizeof(SResultRow); } SResultRowPool* initResultRowPool(size_t size) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 1f5d0cd059..9a81d2fe6b 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tarray.h" +#include #include "dataSinkMgt.h" #include "exception.h" +#include "os.h" +#include "tarray.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" @@ -87,7 +88,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ *handle = (*pTask)->dsHandle; - _error: +_error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -141,6 +142,11 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); + // todo: remove it. + if (tinfo == NULL) { + return TSDB_CODE_SUCCESS; + } + *pRes = NULL; int64_t curOwner = 0; @@ -272,6 +278,19 @@ int32_t qKillTask(qTaskInfo_t qinfo) { return TSDB_CODE_SUCCESS; } +int32_t qAsyncKillTask(qTaskInfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("QInfo:0x%"PRIx64" query async killed", pQInfo->qId); + setQueryKilled(pQInfo); + + return TSDB_CODE_SUCCESS; +} + int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b6df0c527e..dc4d9c7238 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,11 +12,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "tmsg.h" -#include "tglobal.h" -#include "ttime.h" +#include "parser.h" +#include "tq.h" #include "exception.h" +#include "os.h" +#include "tglobal.h" +#include "tmsg.h" +#include "ttime.h" #include "executorimpl.h" #include "function.h" @@ -176,7 +178,7 @@ static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); +static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static bool hasMainOutput(STaskAttr *pQueryAttr); @@ -309,6 +311,31 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO return res; } +SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { + const static int32_t minSize = 8; + + size_t numOfOutput = taosArrayGetSize(pExprInfo); + + SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = numOfOutput; + res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {{0}}; + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + + idata.info.type = pExpr->base.resSchema.type; + idata.info.bytes = pExpr->base.resSchema.bytes; + idata.info.colId = pExpr->base.resSchema.colId; + + int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform + taosArrayPush(res->pDataBlock, &idata); + } + + return res; +} + void* destroyOutputBuf(SSDataBlock* pBlock) { if (pBlock == NULL) { return NULL; @@ -357,8 +384,8 @@ static bool isProjQuery(STaskAttr *pQueryAttr) { return true; } -static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { - if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +static bool hasNull(SColumn* pColumn, SColumnDataAgg *pStatis) { + if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { return false; } @@ -369,7 +396,7 @@ static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { return true; } -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, STaskRuntimeEnv* pRuntimeEnv) { +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) { // more than the capacity, reallocate the resources if (pResultRowInfo->size < pResultRowInfo->capacity) { return; @@ -384,7 +411,7 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, STaskRuntime char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); } pResultRowInfo->pResult = (SResultRow **)t; @@ -473,7 +500,7 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR } if (!existed) { - prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); +// prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); SResultRow *pResult = NULL; if (p1 == NULL) { @@ -507,6 +534,80 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR return pResultRowInfo->pResult[pResultRowInfo->curPos]; } +static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, + bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggOperatorInfo* pAggInfo) { + bool existed = false; + SET_RES_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tableGroupId); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (isIntervalQuery) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + if (pResultRowInfo->size == 0) { + existed = false; + assert(pResultRowInfo->curPos == -1); + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + pResultRowInfo->curPos = 0; + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); + int64_t* index = taosHashGet(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (index != NULL) { + pResultRowInfo->curPos = (int32_t) *index; + existed = true; + } else { + existed = false; + } + } + } + } else { + // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + if (p1 != NULL) { + return *p1; + } + } + + if (!existed) { + prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); + + SResultRow *pResult = NULL; + if (p1 == NULL) { + pResult = getNewResultRow(pAggInfo->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; + taosArrayPush(pAggInfo->pResultRowArrayList, &cell); + } else { + pResult = *p1; + } + + pResultRowInfo->curPos = pResultRowInfo->size; + pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + + int64_t index = pResultRowInfo->curPos; + SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); + taosHashPut(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + } + + // too many time window in query + if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + + return pResultRowInfo->pResult[pResultRowInfo->curPos]; +} + static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); @@ -1023,43 +1124,52 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pC } void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - if (pCtx[0].functionId == FUNCTION_ARITHM) { +// if (pCtx[0].functionId == FUNCTION_ARITHM) { // SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; // if (pSupport->colList == NULL) { // doSetInputDataBlock(pOperator, pCtx, pBlock, order); // } else { // doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); // } - } else { +// } else { if (pBlock->pDataBlock != NULL) { doSetInputDataBlock(pOperator, pCtx, pBlock, order); } else { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } - } +// } } static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { -#if 0 for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; - pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + pCtx[i].currentStage = MAIN_SCAN/*(uint8_t)pOperator->pRuntimeEnv->scanFlag*/; - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns); if (pCtx[i].functionId == FUNCTION_ARITHM) { // setArithParams((SScalarFunctionSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); } else { - SColIndex* pCol = &pOperator->pExpr[i].base.pColumns->info.; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == FUNCTION_BLKINFO) || - (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + uint32_t flag = pOperator->pExpr[i].base.pColumns->flag; + if (TSDB_COL_IS_NORMAL_COL(flag) /*|| (pCtx[i].functionId == FUNCTION_BLKINFO) || + (TSDB_COL_IS_TAG(flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)*/) { + SColumn* pCol = pOperator->pExpr[i].base.pColumns; + if (pCtx[i].columnIndex == -1) { + for(int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + if (pColData->info.colId == pCol->info.colId) { + pCtx[i].columnIndex = j; + break; + } + } + } + + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pCtx[i].columnIndex); // in case of the block distribution query, the inputBytes is not a constant value. pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); + assert(p->info.colId == pCol->info.colId); if (pCtx[i].functionId < 0) { SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); @@ -1070,37 +1180,33 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, // uint32_t status = aAggs[pCtx[i].functionId].status; // if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { -// SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); -// // In case of the top/bottom query again the nest query result, which has no timestamp column -// // don't set the ptsList attribute. -// if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { -// pCtx[i].ptsList = (int64_t*) tsInfo->pData; -// } else { -// pCtx[i].ptsList = NULL; -// } + SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); + // In case of the top/bottom query again the nest query result, which has no timestamp column + // don't set the ptsList attribute. + if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + pCtx[i].ptsList = (int64_t*) tsInfo->pData; + } else { + pCtx[i].ptsList = NULL; + } +// } +// } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { +// SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; +// SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); +// +// pCtx[i].pInput = p->pData; +// assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); +// for(int32_t j = 0; j < pBlock->info.rows; ++j) { +// char* dst = p->pData + j * p->info.bytes; +// taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); // } - } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); - - pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); - for(int32_t j = 0; j < pBlock->info.rows; ++j) { - char* dst = p->pData + j * p->info.bytes; - taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); - } } } } -#endif - } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { + if (functionNeedToExecute(NULL, &pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct pCtx[k].fpSet->addInput(&pCtx[k]); } @@ -1719,7 +1825,6 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // in case of timestamp column, always generated results. int32_t functionId = pCtx->functionId; @@ -1732,28 +1837,28 @@ static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx * } if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { - return QUERY_IS_ASC_QUERY(pQueryAttr); +// return QUERY_IS_ASC_QUERY(pQueryAttr); } // denote the order type if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { - return pCtx->param[0].i == pQueryAttr->order.order; +// return pCtx->param[0].i == pQueryAttr->order.order; } // in the reverse table scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv) || - (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { - return false; - } +// if (IS_REVERSE_SCAN(pRuntimeEnv) || +// (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { +// return false; +// } return true; } -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { +void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { SColumnDataAgg *pAgg = NULL; - if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - pAgg = &pSDataBlock->pBlockAgg[pColIndex->colIndex]; + if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColumn->flag)) { + pAgg = &pSDataBlock->pBlockAgg[pCtx->columnIndex]; pCtx->agg = *pAgg; pCtx->isAggSet = true; @@ -1762,10 +1867,10 @@ void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColInde pCtx->isAggSet = false; } - pCtx->hasNull = hasNull(pColIndex, pAgg); + pCtx->hasNull = hasNull(pColumn, pAgg); // set the statistics data for primary time stamp column - if (pCtx->functionId == FUNCTION_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pCtx->functionId == FUNCTION_SPREAD && pColumn->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pCtx->isAggSet = true; pCtx->agg.min = pSDataBlock->info.window.skey; pCtx->agg.max = pSDataBlock->info.window.ekey; @@ -1918,7 +2023,9 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { +static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { + size_t numOfOutput = taosArrayGetSize(pExprInfo); + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); if (pFuncCtx == NULL) { return NULL; @@ -1931,8 +2038,11 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOu } for (int32_t i = 0; i < numOfOutput; ++i) { - SSqlExpr *pSqlExpr = &pExpr[i].base; + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + + SSqlExpr *pSqlExpr = &pExpr->base; SQLFunctionCtx* pCtx = &pFuncCtx[i]; + #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -1943,15 +2053,16 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOu pCtx->requireNull = false; } #endif -// pCtx->inputBytes = pSqlExpr->colBytes; +// pCtx->inputBytes = pSqlExpr->; // pCtx->inputType = pSqlExpr->colType; pCtx->ptsOutputBuf = NULL; - + pCtx->fpSet = fpSet; + pCtx->columnIndex = -1; pCtx->resDataInfo.bytes = pSqlExpr->resSchema.bytes; pCtx->resDataInfo.type = pSqlExpr->resSchema.type; -// pCtx->order = pQueryAttr->order.order; + pCtx->order = TSDB_ORDER_ASC; // pCtx->functionId = pSqlExpr->functionId; // pCtx->stableQuery = pQueryAttr->stableQuery; pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes; @@ -2007,12 +2118,12 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOu } } -// for(int32_t i = 1; i < numOfOutput; ++i) { -// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes); -// } + for(int32_t i = 1; i < numOfOutput; ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i - 1); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr->base.interBytes); + } setCtxTagColumnInfo(pFuncCtx, numOfOutput); - return pFuncCtx; } @@ -2044,7 +2155,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); - pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); +// pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); @@ -3457,6 +3568,43 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } +void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t stage, SExecTaskInfo* pTaskInfo) { + SOptrBasicInfo *pInfo = &pAggInfo->binfo; + + SQLFunctionCtx* pCtx = pInfo->pCtx; + SSDataBlock* pDataBlock = pInfo->pRes; + int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + + int64_t tid = 0; + pAggInfo->keyBuf = realloc(pAggInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); + SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid, pTaskInfo, false, pAggInfo); + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); + cleanupResultRowEntry(pEntry); + + pCtx[i].resultInfo = pEntry; + pCtx[i].pOutput = pData->pData; + pCtx[i].currentStage = stage; + assert(pCtx[i].pOutput != NULL); + + // set the timestamp output buffer for top/bottom/diff query + int32_t fid = pCtx[i].functionId; + if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { + if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + } + + initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); +} + void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { SSDataBlock* pDataBlock = pBInfo->pRes; @@ -3579,49 +3727,49 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfOutput = pOperator->numOfOutput; - if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { - // for each group result, call the finalize function for each column - if (pQueryAttr->groupbyColumn) { - closeAllResultRows(pResultRowInfo); - } - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *buf = pResultRowInfo->pResult[i]; - if (!isResultRowClosed(pResultRowInfo, i)) { - continue; - } - - setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); - - for (int32_t j = 0; j < numOfOutput; ++j) { -// pCtx[j].startTs = buf->win.skey; -// if (pCtx[j].functionId < 0) { -// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); -// } else { -// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); -// } - } - - - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); - } - - } else { +// if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { +// // for each group result, call the finalize function for each column +// if (pQueryAttr->groupbyColumn) { +// closeAllResultRows(pResultRowInfo); +// } +// +// for (int32_t i = 0; i < pResultRowInfo->size; ++i) { +// SResultRow *buf = pResultRowInfo->pResult[i]; +// if (!isResultRowClosed(pResultRowInfo, i)) { +// continue; +// } +// +// setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); +// +// for (int32_t j = 0; j < numOfOutput; ++j) { +//// pCtx[j].startTs = buf->win.skey; +//// if (pCtx[j].functionId < 0) { +//// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); +//// } else { +//// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); +//// } +// } +// +// +// /* +// * set the number of output results for group by normal columns, the number of output rows usually is 1 except +// * the top and bottom query +// */ +// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); +// } +// +// } else { for (int32_t j = 0; j < numOfOutput; ++j) { // if (pCtx[j].functionId < 0) { // doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); // } else { -// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); + pCtx[j].fpSet->finalize(&pCtx[j]); // } } - } +// } } static bool hasMainOutput(STaskAttr *pQueryAttr) { @@ -4558,6 +4706,7 @@ void appendDownstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); +void createResultBlock(const SArray* pExprInfo, SExchangeInfo* pInfo, const SOperatorInfo* pOperator, size_t size); static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 @@ -4913,58 +5062,164 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { #endif } -int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) { +static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*)param; + // NOTE: this operator never check if current status is done or not + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamBlockScanInfo* pInfo = pOperator->info; + + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + while (tqNextDataBlock(pInfo->readerHandle)) { + pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + terrno = pTaskInfo->code; + return NULL; + } + + if (pBlockInfo->rows == 0) { + return NULL; + } + + pInfo->pRes->pDataBlock = tqRetrieveDataBlock(pInfo->readerHandle); + if (pInfo->pRes->pDataBlock == NULL) { + // TODO add log + pTaskInfo->code = terrno; + return NULL; + } + + break; + } + + // record the scan action. + pInfo->numOfExec++; + pInfo->numOfRows += pBlockInfo->rows; + + return (pBlockInfo->rows == 0)? NULL:pInfo->pRes; +} + +int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) { + SExchangeInfo* pEx = (SExchangeInfo*) param; + pEx->pRsp = pMsg->pData; + + pEx->pRsp->numOfRows = htonl(pEx->pRsp->numOfRows); + pEx->pRsp->useconds = htobe64(pEx->pRsp->useconds); + pEx->pRsp->compLen = htonl(pEx->pRsp->compLen); + + tsem_post(&pEx->ready); +} + +static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->msgInfo.pData); + tfree(pMsgBody); +} + +void processRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SMsgSendInfo *pSendInfo = (SMsgSendInfo *) pMsg->ahandle; + assert(pMsg->ahandle != NULL); + + SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; + + if (pMsg->contLen > 0) { + buf.pData = calloc(1, pMsg->contLen); + if (buf.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(buf.pData, pMsg->pCont, pMsg->contLen); + } + } + + pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); + rpcFreeCont(pMsg->pCont); + destroySendMsgInfo(pSendInfo); } static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; + SOperatorInfo *pOperator = (SOperatorInfo*) param; SExchangeInfo *pExchangeInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; *newgroup = false; + if (pExchangeInfo->pRsp != NULL && pExchangeInfo->pRsp->completed == 1) { + return NULL; + } SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { // todo handle malloc error - + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _error; } - SEpSet epSet; + SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, 0); + SEpSet epSet = {0}; - int64_t sId = -1, queryId = 0, taskId = 1, vgId = 1; - pMsg->header.vgId = htonl(vgId); + epSet.numOfEps = pSource->addr.numOfEps; + epSet.port[0] = pSource->addr.epAddr[0].port; + tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); - pMsg->sId = htobe64(sId); - pMsg->taskId = htobe64(taskId); - pMsg->queryId = htobe64(queryId); + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", queryId, taskId, (int32_t)sizeof(SMsgSendInfo)); + qError("QID:%"PRIx64" calloc %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _error; } - pMsgSendInfo->param = NULL; + pMsgSendInfo->param = pExchangeInfo; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); pMsgSendInfo->msgType = TDMT_VND_FETCH; pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; - void* pTransporter = NULL; - int32_t code = asyncSendMsgToServer(pTransporter, &epSet, &transporterId, pMsgSendInfo); + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pExchangeInfo->ready); - printf("abc\n"); - getchar(); + if (pExchangeInfo->pRsp->numOfRows == 0) { + return NULL; + } - // add it into the sink node + SSDataBlock* pRes = pExchangeInfo->pResult; + char* pData = pExchangeInfo->pRsp->data; + for(int32_t i = 0; i < pOperator->numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); + char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pExchangeInfo->pRsp->numOfRows); + if (tmp == NULL) { + goto _error; + } + + size_t len = pExchangeInfo->pRsp->numOfRows * pColInfoData->info.bytes; + memcpy(tmp, pData, len); + + pColInfoData->pData = tmp; + pData += len; + } + + pRes->info.numOfCols = pOperator->numOfOutput; + pRes->info.rows = pExchangeInfo->pRsp->numOfRows; + + return pExchangeInfo->pResult; + + _error: + tfree(pMsg); + tfree(pMsgSendInfo); + + terrno = pTaskInfo->code; + return NULL; } -SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t numOfSources, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { - assert(numOfSources > 0); +static SSDataBlock* createResultDataBlock(const SArray* pExprInfo); +SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -4975,27 +5230,76 @@ SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t n return NULL; } - pInfo->numOfSources = numOfSources; + pInfo->pSources = taosArrayDup(pSources); + assert(taosArrayGetSize(pInfo->pSources) > 0); + + size_t size = taosArrayGetSize(pExprInfo); + pInfo->pResult = createResultDataBlock(pExprInfo); pOperator->name = "ExchangeOperator"; pOperator->operatorType = OP_Exchange; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; + pOperator->numOfOutput = size; pOperator->pRuntimeEnv = NULL; pOperator->exec = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; + { // todo refactor + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRspMsg; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)"root"; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; +// rpcInit.spi = 1; + rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; + + pInfo->pTransporter = rpcOpen(&rpcInit); + if (pInfo->pTransporter == NULL) { + return NULL; // todo + } + } + return pOperator; } +SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { + SSDataBlock* pResBlock = calloc(1, sizeof(SSDataBlock)); + if (pResBlock == NULL) { + return NULL; + } + + size_t numOfCols = taosArrayGetSize(pExprInfo); + pResBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + + SArray* pResult = pResBlock->pDataBlock; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfoData = {0}; + SExprInfo* p = taosArrayGetP(pExprInfo, i); + + SSchema* pSchema = &p->base.resSchema; + colInfoData.info.type = pSchema->type; + colInfoData.info.colId = pSchema->colId; + colInfoData.info.bytes = pSchema->bytes; + + taosArrayPush(pResult, &colInfoData); + } + + return pResBlock; +} + SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { tfree(pInfo); tfree(pOperator); @@ -5016,7 +5320,6 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; pOperator->pTaskInfo = pTaskInfo; @@ -5049,7 +5352,6 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; pOperator->pTaskInfo = pTaskInfo; @@ -5105,6 +5407,29 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } +SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pInfo->readerHandle = pStreamBlockHandle; + + pOperator->name = "StreamBlockScanOperator"; + pOperator->operatorType = OP_StreamBlockScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->exec = doStreamBlockScan; + pOperator->pTaskInfo = pTaskInfo; +} + + void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { assert(pTableScanInfo != NULL && pDownstream != NULL); @@ -5483,11 +5808,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - + int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { @@ -5499,18 +5820,13 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { break; } - if (pRuntimeEnv->current != NULL) { - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - -// if (downstream->operatorType == OP_DataBlocksOptScan) { -// STableScanInfo* pScanInfo = downstream->info; -// order = getTableScanOrder(pScanInfo); +// if (pAggInfo->current != NULL) { +// setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + doAggregateImpl(pOperator, 0, pInfo->pCtx, pBlock); } doSetOperatorCompleted(pOperator); @@ -6397,19 +6713,31 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); -// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfRows = 1;//(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + size_t numOfOutput = taosArrayGetSize(pExprInfo); + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); + + pInfo->pResultRowHashTable = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pInfo->pResultRowListSet = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pInfo->keyBuf = malloc(1024 + sizeof(int64_t) + POINTER_BYTES); // TODO: + pInfo->pool = initResultRowPool(getResultRowSize(pExprInfo)); + pInfo->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); pInfo->seed = rand(); -// setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN); + setDefaultOutputBuf_rv(pInfo, pInfo->seed, MAIN_SCAN, pTaskInfo); + + SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + assignExprInfo(&p[i], pExpr); + } SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; @@ -6417,10 +6745,11 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = pExpr; + pOperator->pExpr = p; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = NULL; + pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; pOperator->cleanup = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); @@ -7353,61 +7682,113 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { return pTaskInfo; } -SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { +static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId); + +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - return createTableScanOperatorInfo(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); + + tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId); + + return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - return createDataBlocksOptScanInfo(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); - } else { - assert(0); + + tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId); + + return createDataBlocksOptScanInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + } else if (pPhyNode->info.type == OP_Exchange) { + SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; + return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); + } + } + + if (pPhyNode->info.type == OP_Aggregate) { + size_t size = taosArrayGetSize(pPhyNode->pChildren); + assert(size == 1); + + for (int32_t i = 0; i < size; ++i) { + SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId); + return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); } } } -int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { +static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId) { STsdbQueryCond cond = {.loadExternalRows = false}; - uint64_t uid = 0; - SPhyNode* pPhyNode = pPlan->pNode; - if (pPhyNode->info.type == OP_TableScan || pPhyNode->info.type == OP_DataBlocksOptScan) { + cond.order = pTableScanNode->scan.order; + cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + cond.twindow = pTableScanNode->window; + cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; - STableScanPhyNode* pTableScanNode = (STableScanPhyNode*) pPhyNode; - uid = pTableScanNode->scan.uid; - cond.order = pTableScanNode->scan.order; - cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - cond.twindow = pTableScanNode->window; - cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; + for (int32_t i = 0; i < cond.numOfCols; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pTableScanNode->scan.node.pTargets, i); + assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE); - for(int32_t i = 0; i < cond.numOfCols; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pTableScanNode->scan.node.pTargets, i); - assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE); - - SSchema* pSchema = pExprInfo->pExpr->pSchema; - cond.colList[i].type = pSchema->type; - cond.colList[i].bytes = pSchema->bytes; - cond.colList[i].colId = pSchema->colId; - } - } else { - assert(0); + SSchema* pSchema = pExprInfo->pExpr->pSchema; + cond.colList[i].type = pSchema->type; + cond.colList[i].bytes = pSchema->bytes; + cond.colList[i].colId = pSchema->colId; } - STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; - SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; - taosArrayPush(pa, &info); + return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, NULL); +} - taosArrayPush(group.pGroupList, &pa); +static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId) { + int32_t code = 0; + STableGroupInfo groupInfo = {0}; - *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); - tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); + uint64_t uid = pTableScanNode->scan.uid; + STimeWindow window = pTableScanNode->window; + int32_t tableType = pTableScanNode->scan.tableType; - (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + if (tableType == TSDB_SUPER_TABLE) { + code = + tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } else { // Create one table group. + groupInfo.numOfTables = 1; + groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES); + + SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; + taosArrayPush(pa, &info); + taosArrayPush(groupInfo.pGroupList, &pa); + } + + if (groupInfo.numOfTables == 0) { + code = 0; + // qDebug("no table qualified for query, reqId:0x%"PRIx64, (*pTask)->id.queryId); + goto _error; + } + + return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId); + _error: + terrno = code; + return NULL; +} + +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { + tsdbReadHandleT tReaderHandle = NULL; + + int32_t code = 0; + uint64_t queryId = pPlan->id.queryId; + + SPhyNode* pPhyNode = pPlan->pNode; + + *pTaskInfo = createExecTaskInfo(queryId); + + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId); if ((*pTaskInfo)->pRoot == NULL) { return terrno; } diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 9192c19951..8de66584c6 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -28,14 +28,8 @@ extern "C" { #include "function.h" #include "tudf.h" -extern SAggFunctionInfo aggFunc[35]; -typedef struct SResultRowEntryInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - uint32_t numOfRes; // num of output result in current buffer -} SResultRowEntryInfo; +extern SAggFunctionInfo aggFunc[35]; #define FUNCSTATE_SO 0x0u #define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 3b3edf1501..cad40b342a 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -4379,6 +4379,24 @@ int32_t functionCompatList[] = { 6, 8, 7, }; +//typedef struct SFunctionFpSet { +// bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment +// void (*addInput)(struct SQLFunctionCtx *pCtx); +// +// // finalizer must be called after all exec has been executed to generated final result. +// void (*finalize)(struct SQLFunctionCtx *pCtx); +// void (*combine)(struct SQLFunctionCtx *pCtx); +//} SFunctionFpSet; + +SFunctionFpSet fpSet[1] = { + { + .init = function_setup, + .addInput = count_function, + .finalize = doFinalizer, + .combine = count_func_merge, + } +}; + SAggFunctionInfo aggFunc[35] = {{ // 0, count function does not invoke the finalize function "count", diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 4f3330b7b3..bbcc654ae2 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -82,7 +82,9 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { return false; } + if (nRead <= 0) { + return false; + } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -108,7 +110,9 @@ class FstReadMemory { StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; - while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { result.push_back((uint64_t)(rt->out.out)); } + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + result.push_back((uint64_t)(rt->out.out)); + } return true; } bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { @@ -184,7 +188,9 @@ void checkFstPerf() { delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); - if (m->init()) { printf("success to init fst read"); } + if (m->init()) { + printf("success to init fst read"); + } Performance_fstReadRecords(m); delete m; } @@ -348,7 +354,9 @@ class TFileObj { tfileReaderDestroy(reader_); reader_ = NULL; } - if (writer_ == NULL) { InitWriter(); } + if (writer_ == NULL) { + InitWriter(); + } return tfileWriterPut(writer_, tv, false); } bool InitWriter() { @@ -388,8 +396,12 @@ class TFileObj { return tfileReaderSearch(reader_, query, result); } ~TFileObj() { - if (writer_) { tfileWriterDestroy(writer_); } - if (reader_) { tfileReaderDestroy(reader_); } + if (writer_) { + tfileWriterDestroy(writer_); + } + if (reader_) { + tfileReaderDestroy(reader_); + } } private: @@ -912,7 +924,8 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { std::string path = "/tmp/cache_and_tfile"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } std::thread threads[NUM_OF_THREAD]; for (int i = 0; i < NUM_OF_THREAD; i++) { @@ -927,14 +940,24 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { TEST_F(IndexEnv2, testIndex_restart) { std::string path = "/tmp/cache_and_tfile"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } + index->SearchOneTarget("tag1", "Hello", 10); + index->SearchOneTarget("tag2", "Test", 10); +} +TEST_F(IndexEnv2, testIndex_restart1) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->ReadMultiMillonData("tag1", "coding"); index->SearchOneTarget("tag1", "Hello", 10); index->SearchOneTarget("tag2", "Test", 10); } TEST_F(IndexEnv2, testIndex_read_performance) { std::string path = "/tmp/cache_and_tfile"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } index->PutOneTarge("tag1", "Hello", 12); index->PutOneTarge("tag1", "Hello", 15); index->ReadMultiMillonData("tag1", "Hello"); @@ -943,17 +966,84 @@ TEST_F(IndexEnv2, testIndex_read_performance) { } TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/multi_tag"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } int64_t st = taosGetTimestampUs(); int32_t num = 1000 * 10000; index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num); std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl; // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); } -TEST_F(IndexEnv2, testLongComVal) { +TEST_F(IndexEnv2, testLongComVal1) { std::string path = "/tmp/long_colVal"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } // gen colVal by randstr std::string randstr = "xxxxxxxxxxxxxxxxx"; index->WriteMultiMillonData("tag1", randstr, 100 * 10000); } + +TEST_F(IndexEnv2, testLongComVal2) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) { + } + // gen colVal by randstr + std::string randstr = "abcccc fdadfafdafda"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); +} +TEST_F(IndexEnv2, testLongComVal3) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) { + } + // gen colVal by randstr + std::string randstr = "Yes, coding and coding and coding"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); +} +TEST_F(IndexEnv2, testLongComVal4) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) { + } + // gen colVal by randstr + std::string randstr = "111111 bac fdadfa"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); +} +TEST_F(IndexEnv2, testIndex_read_performance1) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); + index->ReadMultiMillonData("tag1", "Hello", 1000); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); +} +TEST_F(IndexEnv2, testIndex_read_performance2) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); + index->ReadMultiMillonData("tag1", "Hello", 1000 * 10); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); +} +TEST_F(IndexEnv2, testIndex_read_performance3) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); + index->ReadMultiMillonData("tag1", "Hello", 1000 * 100); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); +} +TEST_F(IndexEnv2, testIndex_read_performance4) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag10", "Hello", 12); + index->PutOneTarge("tag12", "Hello", 15); + index->ReadMultiMillonData("tag10", "Hello", 1000 * 100); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag10", "Hello")); +} diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 798c9bc97f..638c3d11bf 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -30,7 +30,6 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); SArray *getCurrentExprList(SQueryStmtInfo* pQueryInfo); size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo); -SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema); void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t level); void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 512204b40b..8327b7c131 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2080,7 +2080,7 @@ static int32_t setColumnIndex(SQueryStmtInfo* pQueryInfo, SArray* pParamList, SC STableMeta* pTableMeta = getMetaInfo(pQueryInfo, 0)->pTableMeta; if (pParamList == NULL) { // count(*) is equalled to count(primary_timestamp_key) - *index = (SColumnIndex) {0, PRIMARYKEY_TIMESTAMP_COL_ID, false}; + *index = (SColumnIndex) {0, 0, false}; *columnSchema = *(SSchema*) getOneColumnSchema(pTableMeta, index->columnIndex); } else { tSqlExprItem* pParamElem = taosArrayGet(pParamList, 0); @@ -3955,6 +3955,7 @@ int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtI pQueryInfo->pTableMetaInfo[0]->name = *name; pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo[0]->tagColList = taosArrayInit(4, POINTER_BYTES); + strcpy(pQueryInfo->pTableMetaInfo[0]->aliasName, name->tname); code = setTableVgroupList(pCtx, name, &pQueryInfo->pTableMetaInfo[0]->vgroupList); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 955507ae1b..5852678880 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -26,18 +26,26 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out const char* msg4 = "pattern is invalid"; const char* msg5 = "database name is empty"; const char* msg6 = "pattern string is empty"; - + const char* msg7 = "db is not specified"; /* * database prefix in pInfo->pMiscInfo->a[0] * wildcard in like clause in pInfo->pMiscInfo->a[1] */ int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE) { - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); - SArray* array = NULL; SName name = {0}; - tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + + if (pCtx->db == NULL && pShowInfo->prefix.n == 0) { + return buildInvalidOperationMsg(pMsgBuf, msg7); + } + + SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + if (pShowInfo->prefix.n > 0) { + tNameSetDbName(&name, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); + } else { + tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + } char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); @@ -672,10 +680,9 @@ int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseConte serializeVgroupTablesBatchImpl(&tbatch, pBufArray); destroyCreateTbReqBatch(&tbatch); - } else { // it is a child table, created according to a super table code = doCheckAndBuildCreateCTableReq(pCreateTable, pCtx, pMsgBuf, &pBufArray); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { return code; } } @@ -715,6 +722,8 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf* pMsgBuf = &m; + pDcl->epSet = pCtx->mgmtEpSet; + switch (pInfo->type) { case TSDB_SQL_CREATE_USER: case TSDB_SQL_ALTER_USER: { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 4d506b84a0..b8545b7486 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1656,7 +1656,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha } // Remove quotation marks - if (TK_STRING == type) { + if (TSDB_DATA_TYPE_BINARY == type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } @@ -1947,6 +1947,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) { int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; + const char* msg3 = "db is not specified"; int32_t code = TSDB_CODE_SUCCESS; char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, true); @@ -1984,6 +1985,10 @@ int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, strncpy(name, pTableName->z, pTableName->n); strdequote(name); + if (pParseCtx->db == NULL) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); if (code != TSDB_CODE_SUCCESS) { code = buildInvalidOperationMsg(pMsgBuf, msg2); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index be266bd415..4ff8364198 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -106,7 +106,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId); -void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); +void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource); int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len); int32_t stringToSubplan(const char* str, SSubplan** subplan); diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index a0feaca72b..9c9ff8fe2b 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "function.h" #include "os.h" #include "parser.h" @@ -197,23 +198,23 @@ static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, SQ SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); if (!pQueryInfo->info.projectionQuery) { - STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); + SArray* p = pQueryInfo->exprList[0]; // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); + int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SSourceParam param = {0}; - addIntoSourceParam(¶m, NULL, pCol); - SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); - pExpr[i] = p; + pNode->numOfExpr = numOfCols; + pNode->pExpr = taosArrayInit(numOfCols, POINTER_BYTES); + for(int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(p, i); + SColumn* pCol = pExprInfo->base.pColumns; + + SSchema schema = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); + + tExprNode* pExprNode = pExprInfo->pExpr->_function.pChild[0]; + SExprInfo* px = createBinaryExprInfo(pExprNode, &schema); + taosArrayPush(pNode->pExpr, &px); } - - // pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); - tfree(pExpr); } return pNode; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 422233eed7..1e1a97df1f 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -278,7 +278,7 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) { SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode)); node->srcTemplateId = srcTemplateId; - node->pSrcEndPoints = validPointer(taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr))); + node->pSrcEndPoints = validPointer(taosArrayInit(TARRAY_MIN_SIZE, sizeof(SDownstreamSource))); return (SPhyNode*)node; } @@ -409,24 +409,25 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_SUCCESS; } -void setExchangSourceNode(uint64_t templateId, SQueryNodeAddr* pEp, SPhyNode* pNode) { +void setExchangSourceNode(uint64_t templateId, SDownstreamSource *pSource, SPhyNode* pNode) { if (NULL == pNode) { return; } if (OP_Exchange == pNode->info.type) { SExchangePhyNode* pExchange = (SExchangePhyNode*)pNode; if (templateId == pExchange->srcTemplateId) { - taosArrayPush(pExchange->pSrcEndPoints, pEp); + taosArrayPush(pExchange->pSrcEndPoints, pSource); } } + if (pNode->pChildren != NULL) { size_t size = taosArrayGetSize(pNode->pChildren); for(int32_t i = 0; i < size; ++i) { - setExchangSourceNode(templateId, pEp, taosArrayGetP(pNode->pChildren, i)); + setExchangSourceNode(templateId, pSource, taosArrayGetP(pNode->pChildren, i)); } } } -void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* pEp) { - setExchangSourceNode(templateId, pEp, subplan->pNode); +void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { + setExchangSourceNode(templateId, pSource, subplan->pNode); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 04282a2f80..2abb90993b 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -29,6 +29,14 @@ static void copyString(const cJSON* json, const char* name, char* dst) { strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); } +static uint64_t getBigintFromString(const cJSON* json, const char* name) { + char* val = getString(json, name); + uint64_t intVal = strtoul(val, NULL, 10); + tfree(val); + + return intVal; +} + static int64_t getNumber(const cJSON* json, const char* name) { double d = cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); return (int64_t) d; @@ -400,7 +408,7 @@ static bool functionToJson(const void* obj, cJSON* jFunc) { const tExprNode* exprInfo = (const tExprNode*)obj; bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName); if (res && NULL != exprInfo->_function.pChild) { - res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, *(exprInfo->_function.pChild), sizeof(tExprNode*), exprInfo->_function.num); + res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); } return res; } @@ -459,7 +467,7 @@ static const char* jkExprNodeColumn = "Column"; static const char* jkExprNodeValue = "Value"; static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) { - const tExprNode* exprInfo = (const tExprNode*)obj; + const tExprNode* exprInfo = *(const tExprNode**)obj; bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType); if (res) { switch (exprInfo->nodeType) { @@ -547,7 +555,7 @@ static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) { const SExprInfo* exprInfo = (const SExprInfo*)obj; bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base); if (res) { - res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, exprInfo->pExpr); + res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, &exprInfo->pExpr); } return res; } @@ -567,13 +575,13 @@ static const char* jkTimeWindowEndKey = "EndKey"; static bool timeWindowToJson(const void* obj, cJSON* json) { const STimeWindow* win = (const STimeWindow*)obj; - char tmp[32] = {0}; - sprintf(tmp, "%"PRId64, win->skey); + char tmp[40] = {0}; + snprintf(tmp, tListLen(tmp),"%"PRId64, win->skey); bool res = cJSON_AddStringToObject(json, jkTimeWindowStartKey, tmp); if (res) { memset(tmp, 0, tListLen(tmp)); - sprintf(tmp, "%"PRId64, win->ekey); + snprintf(tmp, tListLen(tmp),"%"PRId64, win->ekey); res = cJSON_AddStringToObject(json, jkTimeWindowEndKey, tmp); } return res; @@ -581,12 +589,8 @@ static bool timeWindowToJson(const void* obj, cJSON* json) { static bool timeWindowFromJson(const cJSON* json, void* obj) { STimeWindow* win = (STimeWindow*)obj; - - char* p = getString(json, jkTimeWindowStartKey); - win->skey = strtoll(p, NULL, 10); - - p = getString(json, jkTimeWindowEndKey); - win->ekey = strtoll(p, NULL, 10); + win->skey = getBigintFromString(json, jkTimeWindowStartKey); + win->ekey = getBigintFromString(json, jkTimeWindowEndKey); return true; } @@ -598,14 +602,19 @@ static const char* jkScanNodeTableRevCount = "Reverse"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* pNode = (const SScanPhyNode*)obj; - bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, pNode->uid); + + char uid[40] = {0}; + snprintf(uid, tListLen(uid), "%"PRIu64, pNode->uid); + bool res = cJSON_AddStringToObject(json, jkScanNodeTableId, uid); if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, pNode->tableType); } + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, pNode->order); } + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, pNode->count); } @@ -613,12 +622,14 @@ static bool scanNodeToJson(const void* obj, cJSON* json) { if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableRevCount, pNode->reverse); } + return res; } static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* pNode = (SScanPhyNode*)obj; - pNode->uid = getNumber(json, jkScanNodeTableId); + + pNode->uid = getBigintFromString(json, jkScanNodeTableId); pNode->tableType = getNumber(json, jkScanNodeTableType); pNode->count = getNumber(json, jkScanNodeTableCount); pNode->order = getNumber(json, jkScanNodeTableOrder); @@ -739,40 +750,72 @@ static bool epAddrFromJson(const cJSON* json, void* obj) { return true; } -static const char* jkNodeAddrId = "NodeId"; -static const char* jkNodeAddrInUse = "InUse"; -static const char* jkNodeAddrEpAddrs = "EpAddrs"; +static const char* jkNodeAddrId = "NodeId"; +static const char* jkNodeAddrInUse = "InUse"; +static const char* jkNodeAddrEpAddrs = "Ep"; +static const char* jkNodeAddr = "NodeAddr"; +static const char* jkNodeTaskId = "TaskId"; +static const char* jkNodeTaskSchedId = "SchedId"; + +static bool queryNodeAddrToJson(const void* obj, cJSON* json) { + const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*) obj; + bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId); + + if (res) { + res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->inUse); + } + + if (res) { + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epAddr, sizeof(SEpAddr), pAddr->numOfEps); + } + return res; +} + +static bool queryNodeAddrFromJson(const cJSON* json, void* obj) { + SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj; + + pAddr->nodeId = getNumber(json, jkNodeAddrId); + pAddr->inUse = getNumber(json, jkNodeAddrInUse); + + int32_t numOfEps = 0; + bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epAddr, sizeof(SEpAddr), &numOfEps); + pAddr->numOfEps = numOfEps; + return res; +} static bool nodeAddrToJson(const void* obj, cJSON* json) { - const SQueryNodeAddr* ep = (const SQueryNodeAddr*)obj; - bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, ep->nodeId); + const SDownstreamSource* pSource = (const SDownstreamSource*) obj; + bool res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId); + if (res) { - res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, ep->inUse); + char t[30] = {0}; + snprintf(t, tListLen(t), "%"PRIu64, pSource->schedId); + res = cJSON_AddStringToObject(json, jkNodeTaskSchedId, t); } + if (res) { - res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, ep->epAddr, ep->numOfEps, sizeof(SEpAddr)); + res = addObject(json, jkNodeAddr, queryNodeAddrToJson, &pSource->addr); } return res; } static bool nodeAddrFromJson(const cJSON* json, void* obj) { - SQueryNodeAddr* ep = (SQueryNodeAddr*)obj; - ep->nodeId = getNumber(json, jkNodeAddrId); - ep->inUse = getNumber(json, jkNodeAddrInUse); - int32_t numOfEps = 0; - bool res = fromRawArray(json, jkNodeAddrEpAddrs, nodeAddrFromJson, &ep->epAddr, sizeof(SEpAddr), &numOfEps); - ep->numOfEps = numOfEps; + SDownstreamSource* pSource = (SDownstreamSource*)obj; + pSource->taskId = getNumber(json, jkNodeTaskId); + + pSource->schedId = getBigintFromString(json, jkNodeTaskSchedId); + bool res = fromObject(json, jkNodeAddr, queryNodeAddrFromJson, &pSource->addr, true); return res; } static const char* jkExchangeNodeSrcTemplateId = "SrcTemplateId"; -static const char* jkExchangeNodeSrcEndPoints = "SrcEndPoints"; +static const char* jkExchangeNodeSrcEndPoints = "SrcAddrs"; static bool exchangeNodeToJson(const void* obj, cJSON* json) { const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId); if (res) { - res = addInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints); + res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData, sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints)); } return res; } @@ -780,7 +823,7 @@ static bool exchangeNodeToJson(const void* obj, cJSON* json) { static bool exchangeNodeFromJson(const cJSON* json, void* obj) { SExchangePhyNode* exchange = (SExchangePhyNode*)obj; exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId); - return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, sizeof(SQueryNodeAddr)); + return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, sizeof(SDownstreamSource)); } static bool specificPhyNodeToJson(const void* obj, cJSON* json) { @@ -989,7 +1032,11 @@ static const char* jkIdSubplanId = "SubplanId"; static bool subplanIdToJson(const void* obj, cJSON* jId) { const SSubplanId* id = (const SSubplanId*)obj; - bool res = cJSON_AddNumberToObject(jId, jkIdQueryId, id->queryId); + + char ids[40] = {0}; + snprintf(ids, tListLen(ids), "%"PRIu64, id->queryId); + + bool res = cJSON_AddStringToObject(jId, jkIdQueryId, ids); if (res) { res = cJSON_AddNumberToObject(jId, jkIdTemplateId, id->templateId); } @@ -1001,9 +1048,10 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) { static bool subplanIdFromJson(const cJSON* json, void* obj) { SSubplanId* id = (SSubplanId*)obj; - id->queryId = getNumber(json, jkIdQueryId); + + id->queryId = getBigintFromString(json, jkIdQueryId); id->templateId = getNumber(json, jkIdTemplateId); - id->subplanId = getNumber(json, jkIdSubplanId); + id->subplanId = getNumber(json, jkIdSubplanId); return true; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index bf815b26b2..9b32213ad7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); @@ -87,8 +87,8 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return TSDB_CODE_SUCCESS; } -void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - setSubplanExecutionNode(subplan, templateId, ep); +void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { + setSubplanExecutionNode(subplan, templateId, pSource); } int32_t qSubPlanToString(const SSubplan *subplan, char** str, int32_t* len) { diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index a3db9c6992..9ada451c61 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -1,15 +1,4 @@ aux_source_directory(src QWORKER_SRC) -#add_library(qworker ${QWORKER_SRC}) -#target_include_directories( -# qworker -# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" -# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -#) -# -#target_link_libraries( -# qworker -# PRIVATE os util transport planner qcom executor -#) add_library(qworker STATIC ${QWORKER_SRC}) target_include_directories( @@ -18,11 +7,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -#set_target_properties(qworker PROPERTIES -# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a" -# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker" -# ) - target_link_libraries(qworker PRIVATE os util transport planner qcom executor ) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 913057787e..5f9b33f7e3 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -31,8 +31,6 @@ enum { QW_PHASE_POST_QUERY, QW_PHASE_PRE_CQUERY, QW_PHASE_POST_CQUERY, - QW_PHASE_PRE_SINK, - QW_PHASE_POST_SINK, QW_PHASE_PRE_FETCH, QW_PHASE_POST_FETCH, }; @@ -86,6 +84,7 @@ typedef struct SQWMsg { typedef struct SQWPhaseInput { int8_t status; + int8_t taskType; int32_t code; qTaskInfo_t taskHandle; DataSinkHandle sinkHandle; @@ -93,8 +92,7 @@ typedef struct SQWPhaseInput { typedef struct SQWPhaseOutput { int32_t rspCode; - bool needStop; - bool needRsp; + bool needStop; } SQWPhaseOutput; @@ -105,10 +103,17 @@ typedef struct SQWTaskStatus { typedef struct SQWTaskCtx { SRWLatch lock; - int32_t phase; + int8_t phase; + int8_t taskType; + + void *readyConnection; + void *dropConnection; + void *cancelConnection; - int32_t sinkId; - int32_t readyCode; + bool emptyRes; + int8_t queryContinue; + int8_t queryInQueue; + int32_t rspCode; int8_t events[QW_EVENT_MAX]; @@ -144,7 +149,11 @@ typedef struct SQWorkerMgmt { #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) -#define QW_IN_EXECUTOR(ctx) ((ctx)->phase == QW_PHASE_PRE_QUERY || (ctx)->phase == QW_PHASE_PRE_CQUERY || (ctx)->phase == QW_PHASE_PRE_FETCH || (ctx)->phase == QW_PHASE_PRE_SINK) +#define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) + +#define QW_SET_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) + +#define QW_IN_EXECUTOR(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_FETCH) #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) @@ -166,6 +175,10 @@ typedef struct SQWorkerMgmt { #define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG_E(param) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) + #define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index 3b5f3b1605..7ecc2b2b20 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -23,7 +23,7 @@ extern "C" { #include "qworkerInt.h" #include "dataSinkMgt.h" -int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); +int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg, int8_t taskType); int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index a0beaba61d..566356e255 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -141,7 +141,8 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, u } else if (QW_NOT_EXIST_RET_ERR == nOpt) { QW_RET(TSDB_CODE_QRY_SCH_NOT_EXIST); } else { - assert(0); + QW_TASK_ELOG("unknown notExistOpt:%d", nOpt); + QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } } @@ -254,6 +255,19 @@ int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ return TSDB_CODE_SUCCESS; } +int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); + if (NULL == (*ctx)) { + QW_TASK_ELOG("ctx not in ctxHash, ctxHashSize:%d", taosHashGetSize(mgmt->ctxHash)); + QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, int32_t status, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -268,6 +282,8 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ if (HASH_NODE_EXIST(code)) { if (rwType && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + } else if (ctx) { + QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } else { QW_TASK_ELOG("task ctx already exist, id:%s", id); QW_ERR_RET(TSDB_CODE_QRY_TASK_ALREADY_EXIST); @@ -281,6 +297,8 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ if (rwType && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + } else if (ctx) { + QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } return TSDB_CODE_SUCCESS; @@ -290,39 +308,48 @@ int32_t qwAddTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tI QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, NULL)); } -int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId)] = {0}; - QW_SET_QTID(id, qId, tId); - - *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); - if (NULL == (*ctx)) { - QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); - } - - return TSDB_CODE_SUCCESS; - -} int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWTaskCtx **ctx) { return qwAddTaskCtxImpl(QW_FPARAMS(), rwType, 0, ctx); } +int32_t qwAddGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { + return qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, ctx); +} + + void qwReleaseTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->ctxLock); } +void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { + // RC WARNING + qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); + if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + qDestroyTask(otaskHandle); + } +} -void qwFreeTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx *ctx) { - if (ctx->taskHandle) { - qDestroyTask(ctx->taskHandle); - ctx->taskHandle = NULL; +int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + int32_t code = 0; + // RC WARNING + qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); + if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { + code = qAsyncKillTask(taskHandle); + atomic_store_ptr(&ctx->taskHandle, taskHandle); } - // TODO - if (ctx->sinkHandle) { + QW_RET(code); +} + +void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); + + if (ctx->sinkHandle) { + dsDestroyDataSinker(ctx->sinkHandle); + ctx->sinkHandle = NULL; } } @@ -341,7 +368,7 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t octx = *ctx; if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { - QW_TASK_ELOG("taosHashRemove from ctx hash failed, id:%s", id); + QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); } @@ -352,6 +379,8 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t if (octx.sinkHandle) { dsDestroyDataSinker(octx.sinkHandle); } + + QW_TASK_DLOG_E("task ctx dropped"); return TSDB_CODE_SUCCESS; } @@ -366,23 +395,23 @@ int32_t qwDropTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_SET_QTID(id, qId, tId); if (qwAcquireScheduler(QW_FPARAMS(), QW_WRITE, &sch)) { - QW_TASK_WLOG("scheduler does not exist, id:%s", id); + QW_TASK_WLOG_E("scheduler does not exist"); return TSDB_CODE_SUCCESS; } if (qwAcquireTaskStatus(QW_FPARAMS(), QW_WRITE, sch, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); - QW_TASK_WLOG("task does not exist, id:%s", id); + QW_TASK_WLOG_E("task does not exist"); return TSDB_CODE_SUCCESS; } if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { - QW_TASK_ELOG("taosHashRemove task from hash failed, task:%p", task); + QW_TASK_ELOG_E("taosHashRemove task from hash failed"); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - QW_TASK_DLOG("task dropped, id:%s", id); + QW_TASK_DLOG_E("task status dropped"); _return: @@ -410,70 +439,30 @@ _return: QW_RET(code); } - -int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needRsp) { - int32_t code = 0; - SQWTaskCtx *ctx = NULL; - bool locked = false; - - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already dropping, phase:%d", ctx->phase); - QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); - } - - if (QW_IN_EXECUTOR(ctx)) { - if (ctx->taskHandle) { - QW_ERR_JRET(qKillTask(ctx->taskHandle)); - } - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); - } else if (ctx->phase > 0) { - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - - locked = false; - *needRsp = true; - } - - if (locked) { - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); - } - -_return: - - if (locked) { - QW_UNLOCK(QW_WRITE, &ctx->lock); - } - - if (ctx) { - qwReleaseTaskCtx(QW_READ, mgmt); - } - - QW_RET(code); -} - -int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHandle) { +int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; uint64_t useconds = 0; - - while (qcontinue) { - code = qExecTask(taskHandle, &pRes, &useconds); + int32_t i = 0; + + while (true) { + QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); + + code = qExecTask(*taskHandle, &pRes, &useconds); if (code) { QW_TASK_ELOG("qExecTask failed, code:%x", code); QW_ERR_JRET(code); } if (NULL == pRes) { - QW_TASK_DLOG("query done, useconds:%"PRIu64, useconds); + QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); + dsEndPut(sinkHandle, useconds); + + if (TASK_TYPE_TEMP == taskType) { + qwFreeTaskHandle(QW_FPARAMS(), taskHandle); + } break; } @@ -483,6 +472,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHa QW_TASK_ELOG("dsPutDataBlock failed, code:%x", code); QW_ERR_JRET(code); } + + QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); + + if (!qcontinue) { + break; + } } _return: @@ -497,6 +492,20 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void bool queryEnd = false; int32_t code = 0; + if (ctx->emptyRes) { + QW_TASK_DLOG("query empty result, query end, phase:%d", ctx->phase); + + QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); + + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + + *rspMsg = rsp; + *dataLen = 0; + pOutput->queryEnd = true; + + return TSDB_CODE_SUCCESS; + } + dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); if (len < 0) { @@ -517,9 +526,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - *rspMsg = rsp; - *dataLen = 0; return TSDB_CODE_SUCCESS; @@ -538,15 +545,12 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_TASK_DLOG("task got data in sink, dataLength:%d", len); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - *rspMsg = rsp; pOutput->pData = rsp->data; - code = dsGetDataBlock(ctx->sinkHandle, pOutput); if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x", code); - qwFreeFetchRsp(rsp); QW_ERR_RET(code); } @@ -564,29 +568,40 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void } -int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { +int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; int8_t status = 0; SQWTaskCtx *ctx = NULL; bool locked = false; + bool ctxAcquired = false; + void *readyConnection = NULL; + void *dropConnection = NULL; + void *cancelConnection = NULL; - QW_SCH_TASK_DLOG("handle event at phase %d", phase); + QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); switch (phase) { case QW_PHASE_PRE_QUERY: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->phase = phase; + atomic_store_32(&ctx->phase, phase); + atomic_store_8(&ctx->taskType, input->taskType); - assert(!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_ELOG("task already cancelled at wrong phase, phase:%d", phase); + + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + break; + } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - output->needStop = true; - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it locked = false; @@ -594,7 +609,6 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->needStop = true; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - qwFreeTask(QW_FPARAMS(), ctx); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); @@ -607,7 +621,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_POST_QUERY: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -615,18 +629,23 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S ctx->taskHandle = input->taskHandle; ctx->sinkHandle = input->sinkHandle; - - ctx->readyCode = input->code; - assert(!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)); + if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { + ctx->emptyRes = true; + } + + if (input->code) { + output->rspCode = input->code; + } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { output->needStop = true; QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it locked = false; @@ -640,11 +659,9 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { - output->needRsp = true; + readyConnection = ctx->readyConnection; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - - output->rspCode = input->code; } if (!output->needStop) { @@ -653,13 +670,14 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_PRE_FETCH: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + ctxAcquired = true; QW_LOCK(QW_WRITE, &ctx->lock); locked = true; - ctx->phase = phase; + atomic_store_32(&ctx->phase, phase); if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); @@ -669,13 +687,15 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task is dropping, phase:%d", phase); + QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task is cancelling, phase:%d", phase); + QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { @@ -691,15 +711,75 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->rspCode = TSDB_CODE_QRY_TASK_MSG_ERROR; QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } break; } case QW_PHASE_POST_FETCH: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); locked = true; + if (input->code) { + output->rspCode = input->code; + } + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("start to drop task, phase:%d", phase); + output->needStop = true; + + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("start to cancel task, phase:%d", phase); + output->needStop = true; + + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + cancelConnection = ctx->cancelConnection; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + break; + } + case QW_PHASE_PRE_CQUERY: { + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + ctxAcquired = true; + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + atomic_store_32(&ctx->phase, phase); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); output->needStop = true; @@ -708,37 +788,115 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task is dropping, phase:%d", phase); + QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task is cancelling, phase:%d", phase); + QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); } break; } + case QW_PHASE_POST_CQUERY: { + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + if (input->code) { + output->rspCode = input->code; + } + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("start to drop task, phase:%d", phase); + output->needStop = true; + + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("start to cancel task, phase:%d", phase); + output->needStop = true; + + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + cancelConnection = ctx->cancelConnection; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + break; + } } - _return: + if (output->rspCode) { + QW_SET_RSP_CODE(ctx, output->rspCode); + } + if (locked) { - ctx->phase = phase; + atomic_store_32(&ctx->phase, phase); QW_UNLOCK(QW_WRITE, &ctx->lock); } - if (ctx) { + if (ctxAcquired && ctx) { qwReleaseTaskCtx(QW_READ, mgmt); } + if (readyConnection) { + qwBuildAndSendReadyRsp(readyConnection, output->rspCode); + QW_TASK_DLOG("ready msg rsped, code:%x", output->rspCode); + } + + if (dropConnection) { + qwBuildAndSendDropRsp(dropConnection, output->rspCode); + QW_TASK_DLOG("drop msg rsped, code:%x", output->rspCode); + } + + if (cancelConnection) { + qwBuildAndSendCancelRsp(cancelConnection, output->rspCode); + QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); + } + + QW_SCH_TASK_DLOG("end to handle event at phase %d", phase); + QW_RET(code); } -int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg, int8_t taskType) { int32_t code = 0; bool queryRsped = false; bool needStop = false; @@ -746,6 +904,10 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t rspCode = 0; SQWPhaseInput input = {0}; SQWPhaseOutput output = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + + input.taskType = taskType; QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); @@ -762,22 +924,29 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_TASK_ELOG("task string to subplan failed, code:%x", code); QW_ERR_JRET(code); } - - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); } + + if ((pTaskInfo && NULL == sinkHandle) || (NULL == pTaskInfo && sinkHandle)) { + QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + //TODO OPTIMIZE EMTYP RESULT QUERY RSP TO AVOID FURTHER FETCH - QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, TSDB_CODE_SUCCESS)); + QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, code)); + QW_TASK_DLOG("query msg rsped, code:%d", code); queryRsped = true; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), pTaskInfo, sinkHandle)); - + if (pTaskInfo && sinkHandle) { + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &pTaskInfo, sinkHandle, taskType)); + } + _return: if (code) { @@ -786,6 +955,10 @@ _return: if (!queryRsped) { code = qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); + if (TSDB_CODE_SUCCESS == code) { + QW_TASK_DLOG("query msg rsped, code:%d", rspCode); + } + if (TSDB_CODE_SUCCESS == rspCode && code) { rspCode = code; } @@ -806,10 +979,6 @@ _return: } QW_ERR_RET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); - - if (queryRsped && output.needRsp) { - qwBuildAndSendReadyRsp(qwMsg->connection, output.rspCode); - } QW_RET(rspCode); } @@ -817,24 +986,45 @@ _return: int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { int32_t code = 0; SQWTaskCtx *ctx = NULL; - - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + int8_t phase = 0; + bool needRsp = false; + int32_t rspCode = 0; + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_LOCK(QW_WRITE, &ctx->lock); - if (ctx->phase == QW_PHASE_PRE_QUERY) { + phase = QW_GET_PHASE(ctx); + + if (phase == QW_PHASE_PRE_QUERY) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); - } else if (ctx->phase == QW_PHASE_POST_QUERY) { + ctx->readyConnection = qwMsg->connection; + QW_TASK_DLOG("ready msg not rsped, phase:%d", phase); + } else if (phase == QW_PHASE_POST_QUERY) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - QW_ERR_JRET(qwBuildAndSendReadyRsp(qwMsg->connection, ctx->readyCode)); + needRsp = true; + rspCode = ctx->rspCode; + } else { + QW_TASK_ELOG("invalid phase when got ready msg, phase:%d", phase); + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); + needRsp = true; + rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); } _return: + if (code && ctx) { + QW_SET_RSP_CODE(ctx, code); + } + if (ctx) { QW_UNLOCK(QW_WRITE, &ctx->lock); + } - qwReleaseTaskCtx(QW_READ, mgmt); + if (needRsp) { + qwBuildAndSendReadyRsp(qwMsg->connection, rspCode); + QW_TASK_DLOG("ready msg rsped, code:%x", rspCode); } QW_RET(code); @@ -842,84 +1032,76 @@ _return: int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { + SQWTaskCtx *ctx = NULL; int32_t code = 0; bool queryRsped = false; bool needStop = false; struct SSubplan *plan = NULL; - int32_t rspCode = 0; SQWPhaseInput input = {0}; SQWPhaseOutput output = {0}; - SQWTaskCtx *ctx = NULL; void *rsp = NULL; int32_t dataLen = 0; - - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); - - needStop = output.needStop; - code = output.rspCode; - if (needStop) { - QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_CQUERY); - QW_ERR_JRET(code); - } + do { + QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - qTaskInfo_t taskHandle = ctx->taskHandle; - DataSinkHandle sinkHandle = ctx->sinkHandle; + needStop = output.needStop; + code = output.rspCode; + + if (needStop) { + QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_CQUERY); + QW_ERR_JRET(code); + } - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), taskHandle, sinkHandle)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CQUERY); + atomic_store_8(&ctx->queryInQueue, 0); - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - SOutputData sOutput = {0}; - QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); - - // Note: schedule data sink firstly and will schedule query after it's done - if (sOutput.scheduleJobNo) { - if (sOutput.scheduleJobNo > ctx->sinkId) { - QW_TASK_DLOG("sink need schedule, scheduleJobNo:%d", sOutput.scheduleJobNo); - - ctx->sinkId = sOutput.scheduleJobNo; - QW_ERR_JRET(qwBuildAndSendSchSinkMsg(QW_FPARAMS(), qwMsg->connection)); - } - } else if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { - QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); - - if (!QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY)) { - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY); - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + DataSinkHandle sinkHandle = ctx->sinkHandle; + + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx->taskHandle, sinkHandle, ctx->taskType)); + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + SOutputData sOutput = {0}; + QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { + QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); - QW_ERR_RET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + // RC WARNING + atomic_store_8(&ctx->queryContinue, 1); + } + + if (rsp) { + qwBuildFetchRsp(rsp, &sOutput, dataLen); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); + } else { + atomic_store_8(&ctx->queryContinue, 1); } } - - if (rsp) { - qwBuildFetchRsp(rsp, &sOutput, dataLen); - } - } + _return: -_return: - - qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - if (code) { + if (code && QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); rsp = NULL; qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); - } else if (rsp) { - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, 0); } - } - - QW_RET(rspCode); + + input.code = code; + qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); + + needStop = output.needStop; + code = output.rspCode; + } while ((!needStop) && (0 == code) && atomic_val_compare_exchange_8(&ctx->queryContinue, 1, 0)); + + QW_RET(code); } @@ -956,46 +1138,49 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t if (NULL == rsp) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); + } else { + qwBuildFetchRsp(rsp, &sOutput, dataLen); } - // Note: schedule data sink firstly and will schedule query after it's done - if (sOutput.scheduleJobNo) { - if (sOutput.scheduleJobNo > ctx->sinkId) { - QW_TASK_DLOG("sink need schedule, scheduleJobNo:%d", sOutput.scheduleJobNo); - - ctx->sinkId = sOutput.scheduleJobNo; - - QW_ERR_JRET(qwBuildAndSendSchSinkMsg(QW_FPARAMS(), qwMsg->connection)); - } - } else if ((!sOutput.queryEnd) && (/* DS_BUF_LOW == sOutput.bufStatus || */ DS_BUF_EMPTY == sOutput.bufStatus)) { + if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); - if (!QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY)) { - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY); + QW_LOCK(QW_WRITE, &ctx->lock); + locked = true; + // RC WARNING + if (QW_IN_EXECUTOR(ctx)) { + atomic_store_8(&ctx->queryContinue, 1); + } else if (0 == atomic_load_8(&ctx->queryInQueue)) { QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); - - QW_ERR_RET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); - } - } - if (rsp) { - qwBuildFetchRsp(rsp, &sOutput, dataLen); + atomic_store_8(&ctx->queryInQueue, 1); + + QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + } } _return: + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + input.code = code; + qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); if (code) { qwFreeFetchRsp(rsp); rsp = NULL; - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); + dataLen = 0; + qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); } else if (rsp) { qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); } - QW_RET(code); } @@ -1003,16 +1188,59 @@ _return: int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { int32_t code = 0; bool needRsp = false; - - QW_ERR_JRET(qwDropTask(QW_FPARAMS(), &needRsp)); + SQWTaskCtx *ctx = NULL; + bool locked = false; + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_WRITE, &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropping, phase:%d", ctx->phase); + QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); + } + + if (QW_IN_EXECUTOR(ctx)) { + QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); + + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); + + ctx->dropConnection = qwMsg->connection; + } else if (ctx->phase > 0) { + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + + locked = false; + needRsp = true; + } + + if (!needRsp) { + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + } + _return: + if (code) { + QW_SET_RSP_CODE(ctx, code); + } + + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + if (ctx) { + qwReleaseTaskCtx(QW_WRITE, mgmt); + } + if (TSDB_CODE_SUCCESS != code || needRsp) { QW_ERR_RET(qwBuildAndSendDropRsp(qwMsg->connection, code)); + + QW_TASK_DLOG("drop msg rsped, code:%x", code); } - return TSDB_CODE_SUCCESS; + QW_RET(code); } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) { @@ -1053,10 +1281,10 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == mgmt->ctxHash) { + qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); taosHashCleanup(mgmt->schHash); mgmt->schHash = NULL; tfree(mgmt); - qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index b9fd8e78b6..feb8fd645e 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -229,42 +229,6 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe return TSDB_CODE_SUCCESS; } - -int32_t qwBuildAndSendSchSinkMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection) { - SRpcMsg *pMsg = (SRpcMsg *)connection; - SSinkDataReq * req = (SSinkDataReq *)rpcMallocCont(sizeof(SSinkDataReq)); - if (NULL == req) { - qError("rpcMallocCont %d failed", (int32_t)sizeof(SSinkDataReq)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - req->header.vgId = mgmt->nodeId; - req->sId = sId; - req->queryId = qId; - req->taskId = tId; - - SRpcMsg pNewMsg = { - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, - .msgType = TDMT_VND_SCHEDULE_DATA_SINK, - .pCont = req, - .contLen = sizeof(SSinkDataReq), - .code = 0, - }; - - int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); - if (TSDB_CODE_SUCCESS != code) { - qError("put data sink schedule msg to queue failed, code:%x", code); - rpcFreeCont(req); - QW_ERR_RET(code); - } - - qDebug("put data sink schedule msg to query queue"); - - return TSDB_CODE_SUCCESS; -} - - int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection) { SRpcMsg *pMsg = (SRpcMsg *)connection; SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); @@ -326,7 +290,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_SCH_TASK_DLOG("processQuery start, node:%p", node); - QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg)); + QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); @@ -366,25 +330,6 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } - - -int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ - if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { - return TSDB_CODE_QRY_INVALID_INPUT; - } - - SSinkDataReq *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - qError("invalid sink data msg"); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - //dsScheduleProcess(); - //TODO - - return TSDB_CODE_SUCCESS; -} - int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -429,7 +374,9 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; msg->sId = htobe64(msg->sId); + uint64_t sId = msg->sId; SSchedulerStatusRsp *sStatus = NULL; diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt index 6d755ad487..a464486546 100644 --- a/source/libs/qworker/test/CMakeLists.txt +++ b/source/libs/qworker/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( qworkerTest - PUBLIC os util common transport gtest qcom planner qworker + PUBLIC os util common transport gtest qcom planner qworker executor ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 4962eab460..d1cc9f03d1 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -32,11 +32,75 @@ #include "qworker.h" #include "stub.h" #include "addr_any.h" +#include "executor.h" +#include "dataSinkMgt.h" namespace { -bool testStop = false; +bool qwtTestEnableSleep = true; +bool qwtTestStop = false; +bool qwtTestDeadLoop = true; +int32_t qwtTestMTRunSec = 10; +int32_t qwtTestPrintNum = 100000; +int32_t qwtTestCaseIdx = 0; +int32_t qwtTestCaseNum = 4; + +void qwtInitLogFile() { + const char *defaultLogFileNamePrefix = "taosdlog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } + +} + +void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); + queryMsg->queryId = htobe64(1); + queryMsg->sId = htobe64(1); + queryMsg->taskId = htobe64(1); + queryMsg->contentLen = htonl(100); + queryRpc->pCont = queryMsg; + queryRpc->contLen = sizeof(SSubQueryMsg) + 100; +} + +void qwtBuildReadyReqMsg(SResReadyReq *readyMsg, SRpcMsg *readyRpc) { + readyMsg->sId = htobe64(1); + readyMsg->queryId = htobe64(1); + readyMsg->taskId = htobe64(1); + readyRpc->pCont = readyMsg; + readyRpc->contLen = sizeof(SResReadyReq); +} + +void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { + fetchMsg->sId = htobe64(1); + fetchMsg->queryId = htobe64(1); + fetchMsg->taskId = htobe64(1); + fetchRpc->pCont = fetchMsg; + fetchRpc->contLen = sizeof(SResFetchReq); +} + +void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { + dropMsg->sId = htobe64(1); + dropMsg->queryId = htobe64(1); + dropMsg->taskId = htobe64(1); + dropRpc->pCont = dropMsg; + dropRpc->contLen = sizeof(STaskDropReq); +} + +void qwtBuildStatusReqMsg(SSchTasksStatusReq *statusMsg, SRpcMsg *statusRpc) { + statusMsg->sId = htobe64(1); + statusRpc->pCont = statusMsg; + statusRpc->contLen = sizeof(SSchTasksStatusReq); + statusRpc->msgType = TDMT_VND_TASKS_STATUS; +} int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { return 0; @@ -48,6 +112,7 @@ int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { void qwtRpcSendResponse(const SRpcMsg *pRsp) { +/* if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; printf("task num:%d\n", rsp->num); @@ -56,9 +121,63 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { printf("qId:%"PRIx64",tId:%"PRIx64",status:%d\n", task->queryId, task->taskId, task->status); } } +*/ return; } +int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { + int32_t idx = qwtTestCaseIdx % qwtTestCaseNum; + + if (0 == idx) { + *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; + *handle = (DataSinkHandle)qwtTestCaseIdx+1; + } else if (1 == idx) { + *pTaskInfo = NULL; + *handle = NULL; + } else if (2 == idx) { + *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; + *handle = NULL; + } else if (3 == idx) { + *pTaskInfo = NULL; + *handle = (DataSinkHandle)qwtTestCaseIdx; + } + + ++qwtTestCaseIdx; + + return 0; +} + +int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { + return 0; +} + +int32_t qwtKillTask(qTaskInfo_t qinfo) { + return 0; +} + +void qwtDestroyTask(qTaskInfo_t qHandle) { + +} + + +int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { + return 0; +} + +void qwtEndPut(DataSinkHandle handle, uint64_t useconds) { +} + +void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { +} + +int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { + return 0; +} + +void qwtDestroyDataSinker(DataSinkHandle handle) { + +} + void stubSetStringToPlan() { @@ -74,11 +193,118 @@ void stubSetStringToPlan() { } } +void stubSetExecTask() { + static Stub stub; + stub.set(qExecTask, qwtExecTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qExecTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtExecTask); + } + } +} + + + +void stubSetCreateExecTask() { + static Stub stub; + stub.set(qCreateExecTask, qwtCreateExecTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qCreateExecTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtCreateExecTask); + } + } +} + +void stubSetAsyncKillTask() { + static Stub stub; + stub.set(qAsyncKillTask, qwtKillTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qAsyncKillTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtKillTask); + } + } +} + +void stubSetDestroyTask() { + static Stub stub; + stub.set(qDestroyTask, qwtDestroyTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qDestroyTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtDestroyTask); + } + } +} + + +void stubSetDestroyDataSinker() { + static Stub stub; + stub.set(dsDestroyDataSinker, qwtDestroyDataSinker); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsDestroyDataSinker$", result); + for (const auto& f : result) { + stub.set(f.second, qwtDestroyDataSinker); + } + } +} + +void stubSetGetDataLength() { + static Stub stub; + stub.set(dsGetDataLength, qwtGetDataLength); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsGetDataLength$", result); + for (const auto& f : result) { + stub.set(f.second, qwtGetDataLength); + } + } +} + +void stubSetEndPut() { + static Stub stub; + stub.set(dsEndPut, qwtEndPut); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsEndPut$", result); + for (const auto& f : result) { + stub.set(f.second, qwtEndPut); + } + } +} + +void stubSetPutDataBlock() { + static Stub stub; + stub.set(dsPutDataBlock, qwtPutDataBlock); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsPutDataBlock$", result); + for (const auto& f : result) { + stub.set(f.second, qwtPutDataBlock); + } + } +} + void stubSetRpcSendResponse() { static Stub stub; stub.set(rpcSendResponse, qwtRpcSendResponse); { - AddrAny any("libplanner.so"); + AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendResponse$", result); for (const auto& f : result) { @@ -87,24 +313,35 @@ void stubSetRpcSendResponse() { } } +void stubSetGetDataBlock() { + static Stub stub; + stub.set(dsGetDataBlock, qwtGetDataBlock); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsGetDataBlock$", result); + for (const auto& f : result) { + stub.set(f.second, qwtGetDataBlock); + } + } +} + + void *queryThread(void *param) { SRpcMsg queryRpc = {0}; int32_t code = 0; uint32_t n = 0; void *mockPointer = (void *)0x1; void *mgmt = param; - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - while (!testStop) { - qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + while (!qwtTestStop) { + qwtBuildQueryReqMsg(&queryRpc); + qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + free(queryRpc.pCont); + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("query:%d\n", n); } } @@ -119,16 +356,14 @@ void *readyThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; SResReadyReq readyMsg = {0}; - readyMsg.sId = htobe64(1); - readyMsg.queryId = htobe64(1); - readyMsg.taskId = htobe64(1); - readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyReq); - while (!testStop) { + while (!qwtTestStop) { + qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("ready:%d\n", n); } } @@ -143,16 +378,14 @@ void *fetchThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; SResFetchReq fetchMsg = {0}; - fetchMsg.sId = htobe64(1); - fetchMsg.queryId = htobe64(1); - fetchMsg.taskId = htobe64(1); - fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchReq); - while (!testStop) { + while (!qwtTestStop) { + qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("fetch:%d\n", n); } } @@ -167,16 +400,14 @@ void *dropThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - while (!testStop) { + while (!qwtTestStop) { + qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("drop:%d\n", n); } } @@ -191,16 +422,14 @@ void *statusThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; - while (!testStop) { - statusMsg.sId = htobe64(1); + while (!qwtTestStop) { + qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("status:%d\n", n); } } @@ -209,6 +438,35 @@ void *statusThread(void *param) { } +void *controlThread(void *param) { + SRpcMsg queryRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + + while (!qwtTestStop) { + qwtBuildQueryReqMsg(&queryRpc); + qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + free(queryRpc.pCont); + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { + printf("query:%d\n", n); + } + } + + return NULL; +} + +void *queryQueueThread(void *param) { + +} + +void *fetchQueueThread(void *param) { + +} @@ -224,6 +482,8 @@ TEST(seqTest, normalCase) { SRpcMsg fetchRpc = {0}; SRpcMsg dropRpc = {0}; SRpcMsg statusRpc = {0}; + + qwtInitLogFile(); SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); queryMsg->queryId = htobe64(1); @@ -262,6 +522,15 @@ TEST(seqTest, normalCase) { stubSetStringToPlan(); stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -308,6 +577,8 @@ TEST(seqTest, cancelFirst) { SRpcMsg queryRpc = {0}; SRpcMsg dropRpc = {0}; SRpcMsg statusRpc = {0}; + + qwtInitLogFile(); SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); queryMsg->queryId = htobe64(1); @@ -348,7 +619,7 @@ TEST(seqTest, cancelFirst) { ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - ASSERT_EQ(code, 0); + ASSERT_EQ(code, TSDB_CODE_QRY_TASK_DROPPED); statusMsg.sId = htobe64(1); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); @@ -366,44 +637,16 @@ TEST(seqTest, randCase) { SRpcMsg fetchRpc = {0}; SRpcMsg dropRpc = {0}; SRpcMsg statusRpc = {0}; - - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - SResReadyReq readyMsg = {0}; - readyMsg.sId = htobe64(1); - readyMsg.queryId = htobe64(1); - readyMsg.taskId = htobe64(1); - readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyReq); - SResFetchReq fetchMsg = {0}; - fetchMsg.sId = htobe64(1); - fetchMsg.queryId = htobe64(1); - fetchMsg.taskId = htobe64(1); - fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchReq); - STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; + + qwtInitLogFile(); stubSetStringToPlan(); stubSetRpcSendResponse(); + stubSetCreateExecTask(); srand(time(NULL)); @@ -416,20 +659,25 @@ TEST(seqTest, randCase) { int32_t r = rand() % maxr; if (r >= 0 && r < maxr/5) { - printf("Query,%d\n", t++); + printf("Query,%d\n", t++); + qwtBuildQueryReqMsg(&queryRpc); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + free(queryRpc.pCont); } else if (r >= maxr/5 && r < maxr * 2/5) { printf("Ready,%d\n", t++); + qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); + qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); } else if (r >= maxr * 3/5 && r < maxr * 4/5) { printf("Drop,%d\n", t++); + qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); } else if (r >= maxr * 4/5 && r < maxr-1) { printf("Status,%d\n", t++); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); } else { @@ -445,6 +693,8 @@ TEST(seqTest, multithreadRand) { void *mgmt = NULL; int32_t code = 0; void *mockPointer = (void *)0x1; + + qwtInitLogFile(); stubSetStringToPlan(); stubSetRpcSendResponse(); @@ -464,15 +714,69 @@ TEST(seqTest, multithreadRand) { pthread_create(&(t4), &thattr, dropThread, NULL); pthread_create(&(t5), &thattr, statusThread, NULL); - int32_t t = 0; - int32_t maxr = 10001; - sleep(300); - testStop = true; - sleep(1); + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + sleep(3); qWorkerDestroy(&mgmt); } +TEST(rcTest, multithread) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, controlThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, NULL); + pthread_create(&(t3), &thattr, fetchQueueThread, NULL); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + sleep(3); + + qWorkerDestroy(&mgmt); +} + + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 661beee5d5..6b047eb96e 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -29,7 +29,7 @@ extern "C" { #define SCHEDULE_DEFAULT_JOB_NUMBER 1000 #define SCHEDULE_DEFAULT_TASK_NUMBER 1000 -#define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA +#define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA enum { SCH_READ = 1, diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index de295f77c6..d8904cdfa9 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -52,9 +52,9 @@ int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel * pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); - pTask->execAddrs = taosArrayInit(SCH_MAX_CONDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->execAddrs) { - SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CONDIDATE_EP_NUM); + SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -66,7 +66,7 @@ void schFreeTask(SSchTask* pTask) { taosArrayDestroy(pTask->candidateAddrs); } - tfree(pTask->msg); + tfree(pTask->msg); if (pTask->children) { taosArrayDestroy(pTask->children); @@ -408,9 +408,9 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } pTask->candidateIdx = 0; - pTask->candidateAddrs = taosArrayInit(SCH_MAX_CONDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + pTask->candidateAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { - SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CONDIDATE_EP_NUM); + SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -430,10 +430,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CONDIDATE_EP_NUM; ++i) { + for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); - if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -443,12 +443,12 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } if (addNum <= 0) { - SCH_TASK_ELOG("no available execNode as condidate addr, nodeNum:%d", nodeNum); + SCH_TASK_ELOG("no available execNode as candidate addr, nodeNum:%d", nodeNum); return TSDB_CODE_QRY_INVALID_INPUT; } /* - for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CONDIDATE_EP_NUM; ++i) { + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; @@ -767,7 +767,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } /* - if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CONDIDATE_EP_NUM) { + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; @@ -782,7 +782,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); - qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->succeedAddr); + SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; + qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); if (SCH_TASK_READY_TO_LUNCH(par)) { @@ -853,7 +854,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SResReadyRsp *rsp = (SResReadyRsp *)msg; if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -879,7 +880,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); } - + SCH_ERR_JRET(schProcessOnDataFetched(pJob)); break; @@ -1101,6 +1102,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskType = TASK_TYPE_TEMP; pMsg->contentLen = htonl(pTask->msgLen); memcpy(pMsg->msg, pTask->msg, pTask->msgLen); break; @@ -1214,6 +1216,8 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("subplanToString error, code:%x, msg:%p, len:%d", code, pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); } + +// printf("physical plan:%s\n", pTask->msg); } SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); @@ -1477,13 +1481,14 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSubQueryMsg *pMsg = msg; + SSubQueryMsg *pMsg = (SSubQueryMsg*) msg; pMsg->header.vgId = htonl(tInfo.addr.nodeId); pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(plan->id.queryId); pMsg->taskId = htobe64(schGenUUID()); + pMsg->taskType = TASK_TYPE_PERSISTENT; pMsg->contentLen = htonl(msgLen); memcpy(pMsg->msg, msg, msgLen); @@ -1512,7 +1517,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { } int32_t code = 0; - + *dst = taosArrayInit(copyNum, sizeof(STaskInfo)); if (NULL == *dst) { qError("taosArrayInit %d taskInfo failed", copyNum); @@ -1523,7 +1528,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { STaskInfo info = {0}; info.addr = src->addr; - + for (int32_t i = 0; i < copyNum; ++i) { info.msg = malloc(msgSize); if (NULL == info.msg) { @@ -1534,7 +1539,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { memcpy(info.msg, src->msg, msgSize); info.msg->taskId = schGenUUID(); - + if (NULL == taosArrayPush(*dst, &info)) { qError("taosArrayPush failed, idx:%d", i); free(info.msg); @@ -1548,7 +1553,7 @@ _return: schedulerFreeTaskList(*dst); *dst = NULL; - + SCH_RET(code); } @@ -1668,7 +1673,7 @@ void scheduleFreeJob(void *job) { schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); setJobFree = true; } - + usleep(1); } else { assert(0); diff --git a/source/libs/tfs/CMakeLists.txt b/source/libs/tfs/CMakeLists.txt index 1b6f662507..607ccd4c48 100644 --- a/source/libs/tfs/CMakeLists.txt +++ b/source/libs/tfs/CMakeLists.txt @@ -6,4 +6,8 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(tfs os util common) \ No newline at end of file +target_link_libraries(tfs os util common) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index c88a2a4ea8..cfc246f07b 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -22,7 +22,6 @@ #include "taoserror.h" #include "tcoding.h" #include "tfs.h" -#include "tglobal.h" #include "thash.h" #include "tlog.h" diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index be411744cf..623293f82b 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -141,6 +141,7 @@ const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId) { return TFS_DISK_AT(pTfs void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + if (pDisk == NULL) return; pFile->did = diskId; tstrncpy(pFile->rname, rname, TSDB_FILENAME_LEN); @@ -197,9 +198,7 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } -int32_t tfsRemoveFile(const STfsFile *pFile) { - return remove(pFile->aname); -} +int32_t tfsRemoveFile(const STfsFile *pFile) { return remove(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { return taosCopyFile(pFile1->aname, pFile2->aname); @@ -291,6 +290,8 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); if (taosRenameFile(oaname, naname) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); return -1; } } @@ -330,7 +331,12 @@ const STfsFile *tfsReaddir(STfsDir *pDir) { // Skip . and .. if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; - snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); + if (pDir->dirname == NULL || pDir->dirname[0] == 0) { + snprintf(bname, TMPNAME_LEN * 2, "%s", dp->d_name); + } else { + snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); + } + tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname); return &pDir->tfile; } @@ -402,8 +408,7 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { } if (tfsFormatDir(pCfg->dir, dirName) < 0) { - fError("failed to mount %s to FS since invalid dir format", pCfg->dir); - terrno = TSDB_CODE_FS_INVLD_CFG; + fError("failed to mount %s to FS since %s", pCfg->dir, terrstr()); return -1; } @@ -501,7 +506,11 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) { pDir->did.level = pDisk->level; pDir->did.id = pDisk->id; - snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + if (pDisk->path == NULL || pDisk->path[0] == 0) { + snprintf(adir, TMPNAME_LEN * 2, "%s", pDir->dirname); + } else { + snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + } pDir->dir = opendir(adir); if (pDir->dir != NULL) break; } diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt new file mode 100644 index 0000000000..ee28dcf723 --- /dev/null +++ b/source/libs/tfs/test/CMakeLists.txt @@ -0,0 +1,14 @@ +enable_testing() + +aux_source_directory(. TFS_TEST_SRC) +add_executable(tfs_test ${TFS_TEST_SRC}) +target_link_libraries( + tfs_test + PUBLIC tfs + PUBLIC gtest_main +) + +add_test( + NAME tfs_test + COMMAND tfs_test +) diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp new file mode 100644 index 0000000000..abc00c5cd9 --- /dev/null +++ b/source/libs/tfs/test/tfsTest.cpp @@ -0,0 +1,293 @@ +/** + * @file tfsTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief TFS module tests + * @version 1.0 + * @date 2022-01-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "os.h" + +#include "tfs.h" + +class TfsTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { root = "/tmp/tfsTest"; } + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + + static const char *root; +}; + +const char *TfsTest::root; + +TEST_F(TfsTest, 01_Open_Close) { + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosRemoveDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_EQ(pTfs, nullptr); + + taosMkDir(root); + pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + tfsUpdateSize(pTfs); + SDiskSize size = tfsGetSize(pTfs); + + EXPECT_GT(size.avail, 0); + EXPECT_GT(size.used, 0); + EXPECT_GT(size.total, size.avail); + EXPECT_GT(size.total, size.used); + + tfsClose(pTfs); +} + +TEST_F(TfsTest, 02_AllocDisk) { + int32_t code = 0; + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosRemoveDir(root); + taosMkDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + SDiskID did; + did.id = 0; + did.level = 0; + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + did.id = 1; + did.level = 1; + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + did.id = 1; + did.level = 2; + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + did.id = 1; + did.level = 3; + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + const char *primary = tfsGetPrimaryPath(pTfs); + EXPECT_STREQ(primary, root); + + const char *path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root); + + tfsClose(pTfs); +} + +TEST_F(TfsTest, 03_Dir) { + int32_t code = 0; + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosRemoveDir(root); + taosMkDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + char p1[] = "p1"; + char ap1[128] = {0}; + snprintf(ap1, 128, "%s%s%s", root, TD_DIRSEP, p1); + EXPECT_NE(taosDirExist(ap1), 0); + EXPECT_EQ(tfsMkdir(pTfs, p1), 0); + EXPECT_EQ(taosDirExist(ap1), 0); + + char p2[] = "p2"; + char ap2[128] = {0}; + snprintf(ap2, 128, "%s%s%s", root, TD_DIRSEP, p2); + SDiskID did = {0}; + EXPECT_NE(taosDirExist(ap2), 0); + EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); + EXPECT_EQ(taosDirExist(ap2), 0); + + char p3[] = "p3/p2/p1/p0"; + char ap3[128] = {0}; + snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3); + EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(tfsMkdir(pTfs, p3), 0); + EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); + EXPECT_EQ(taosDirExist(ap3), 0); + + EXPECT_EQ(tfsRmdir(pTfs, p3), 0); + EXPECT_NE(taosDirExist(ap3), 0); + + char p45[] = "p5"; + char p44[] = "p4"; + char p4[] = "p4/p2/p1/p0"; + char ap4[128] = {0}; + snprintf(ap4, 128, "%s%s%s", root, TD_DIRSEP, p4); + + EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); + EXPECT_EQ(taosDirExist(ap4), 0); + EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRmdir(pTfs, p4), 0); + EXPECT_NE(taosDirExist(ap4), 0); + + tfsClose(pTfs); +} + +TEST_F(TfsTest, 04_File) { + int32_t code = 0; + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosRemoveDir(root); + taosMkDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + STfsFile file0; + STfsFile file1; + STfsFile file2; + STfsFile file3; + STfsFile file4; + SDiskID did0 = {0}; + SDiskID did1 = {0}; + SDiskID did2 = {0}; + SDiskID did3 = {0}; + SDiskID did4 = {0}; + did3.id = 1; + did4.level = 1; + tfsInitFile(pTfs, &file0, did0, "fname"); + tfsInitFile(pTfs, &file1, did1, "fname"); + tfsInitFile(pTfs, &file2, did2, "fnamex"); + tfsInitFile(pTfs, &file3, did3, "fname"); + tfsInitFile(pTfs, &file4, did4, "fname"); + + EXPECT_TRUE(tfsIsSameFile(&file0, &file1)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file2)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file3)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file4)); + + { + int32_t size = 1024; + void *ret = malloc(size + sizeof(size_t)); + *(size_t *)ret = size; + void *buf = (void *)((char *)ret + sizeof(size_t)); + + file0.did.id = 0; + file0.did.level = 0; + int32_t len = tfsEncodeFile((void **)&buf, &file0); + EXPECT_EQ(len, 8); + + STfsFile outfile = {0}; + char *outbuf = (char *)tfsDecodeFile(pTfs, (void *)((char *)buf - len), &outfile); + int32_t decodeLen = (outbuf - (char *)buf); + + EXPECT_EQ(outfile.did.id, 0); + EXPECT_EQ(outfile.did.level, 0); + EXPECT_STREQ(outfile.aname, file0.aname); + EXPECT_STREQ(outfile.rname, "fname"); + EXPECT_EQ(outfile.pTfs, pTfs); + } + + { + char n1[] = "t3/t1.json"; + char n2[] = "t3/t2.json"; + STfsFile f1 = {0}; + STfsFile f2 = {0}; + SDiskID did; + did.id = 0; + did.level = 0; + + tfsInitFile(pTfs, &f1, did, n1); + tfsInitFile(pTfs, &f2, did, n2); + + EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); + + FILE *fp = fopen(f1.aname, "w"); + ASSERT_NE(fp, nullptr); + fwrite("12345678", 1, 5, fp); + fclose(fp); + + char base[128] = {0}; + tfsBasename(&f1, base); + char dir[128] = {0}; + tfsDirname(&f1, dir); + + EXPECT_STREQ(base, "t1.json"); + + char fulldir[128]; + snprintf(fulldir, 128, "%s%s%s", root, TD_DIRSEP, "t3"); + EXPECT_STREQ(dir, fulldir); + + EXPECT_NE(tfsCopyFile(&f1, &f2), 0); + + char af2[128] = {0}; + snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); + EXPECT_EQ(taosDirExist(af2), 0); + tfsRemoveFile(&f2); + EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(tfsCopyFile(&f1, &f2), 0); + + { + STfsDir *pDir = tfsOpendir(pTfs, ""); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_STREQ(pf1->rname, "t3"); + EXPECT_EQ(pf1->did.id, 0); + EXPECT_EQ(pf1->did.level, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_EQ(pf2, nullptr); + + tfsClosedir(pDir); + } + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_EQ(pf1->did.id, 0); + EXPECT_EQ(pf1->did.level, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_NE(pf2, nullptr); + + const STfsFile *pf3 = tfsReaddir(pDir); + EXPECT_EQ(pf3, nullptr); + + tfsClosedir(pDir); + } + } + + tfsClose(pTfs); +} \ No newline at end of file diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 4b14f9f2c7..c760acd52e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -102,38 +102,110 @@ typedef void* queue[2]; #define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - struct SRpcConn* pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - SEpSet* pSet; // for synchronous API - char msg[0]; // RpcHead starts from here + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + // struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + // int32_t code; // error code + // int16_t numOfTry; // number of try for different servers + // int8_t oldInUse; // server EP inUse passed by app + // int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + char* ip; + uint32_t port; + // SEpSet* pSet; // for synchronous API } SRpcReqContext; -#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) +typedef struct { + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + // struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + // int32_t code; // error code + // int16_t numOfTry; // number of try for different servers + // int8_t oldInUse; // server EP inUse passed by app + // int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + char* ip; + uint32_t port; + // SEpSet* pSet; // for synchronous API +} STransConnCtx; -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#pragma pack(push, 1) + +typedef struct { + char version : 4; // RPC version + char comp : 4; // compression algorithm, 0:no compression 1:lz4 + char resflag : 2; // reserved bits + char spi : 3; // security parameter index + char encrypt : 3; // encrypt algorithm, 0: no encryption + + uint32_t code; // del later + uint32_t msgType; + int32_t msgLen; + uint8_t content[0]; // message body starts from here +} STransMsgHead; + +typedef struct { + int32_t reserved; + int32_t contLen; +} STransCompMsg; + +typedef struct { + uint32_t timeStamp; + uint8_t auth[TSDB_AUTH_LEN]; +} STransDigestMsg; + +#pragma pack(pop) + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) + +#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) #define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) #define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) +#define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) + +#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead) + sizeof(STransDigestMsg)) +#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) +#define transContFromHead(msg) (msg + sizeof(STransMsgHead)) +#define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead)) +#define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); +#define transIsReq(type) (type & 1U) + int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); +int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +bool transCompressMsg(char* msg, int32_t len, int32_t* flen); +bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); + +void transConnCtxDestroy(STransConnCtx* ctx); + +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; +} SConnBuffer; + #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index e39e0d9273..3c8c922d83 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -45,6 +45,9 @@ extern "C" { void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void taosCloseServer(void* arg); +void taosCloseClient(void* arg); + typedef struct { int sessions; // number of sessions allowed int numOfThreads; // number of threads to process incoming messages diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index cb8ef87b48..4b490936cc 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -17,15 +17,9 @@ #include "transComm.h" -typedef struct SConnBuffer { - char* buf; - int len; - int cap; - int left; -} SConnBuffer; - -void* (*taosHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { taosInitServer, taosInitClient}; +void (*taosCloseHandle[])(void* arg) = {taosCloseServer, taosCloseClient}; void* rpcOpen(const SRpcInit* pInit) { SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); @@ -38,13 +32,18 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->connType = pInit->connType; - pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; } -void rpcClose(void* arg) { return; } +void rpcClose(void* arg) { + SRpcInfo* pRpc = (SRpcInfo*)arg; + (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + free(pRpc); + return; +} void* rpcMallocCont(int contLen) { - int size = contLen + RPC_MSG_OVERHEAD; + int size = contLen + TRANS_MSG_OVERHEAD; char* start = (char*)calloc(1, (size_t)size); if (start == NULL) { @@ -53,7 +52,7 @@ void* rpcMallocCont(int contLen) { } else { tTrace("malloc mem:%p size:%d", start, size); } - return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); + return start + sizeof(STransMsgHead); } void rpcFreeCont(void* cont) { return; } void* rpcReallocCont(void* ptr, int contLen) { return NULL; } @@ -71,6 +70,7 @@ int32_t rpcInit(void) { void rpcCleanup(void) { // impl later + // return; } #endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f197e72ec5..f2d844f73d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -21,25 +21,32 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; uv_write_t* writeReq; + SConnBuffer readBuf; void* data; queue conn; char spi; char secured; + uint64_t expireTime; } SCliConn; + typedef struct SCliMsg { - SRpcReqContext* context; - queue q; - uint64_t st; + STransConnCtx* ctx; + SRpcMsg msg; + queue q; + uint64_t st; } SCliMsg; typedef struct SCliThrdObj { pthread_t thread; uv_loop_t* loop; uv_async_t* cliAsync; // - void* cache; // conn pool + uv_timer_t* pTimer; + void* cache; // conn pool queue msg; pthread_mutex_t msgMtx; - void* shandle; + uint64_t nextTimeout; // next timeout + void* shandle; // + } SCliThrdObj; typedef struct SClientObj { @@ -49,31 +56,188 @@ typedef struct SClientObj { SCliThrdObj** pThreadObj; } SClientObj; +typedef struct SConnList { + queue conn; +} SConnList; + // conn pool +// add expire timeout and capacity limit +static void* connCacheCreate(int size); +static void* connCacheDestroy(void* cache); static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); -static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +// register timer in each thread to clear expire conn +static void clientTimeoutCb(uv_timer_t* handle); +// process data read from server, auth/decompress etc +static void clientProcessData(SCliConn* conn); +// check whether already read complete packet from server +static bool clientReadComplete(SConnBuffer* pBuf); +// alloc buf for read +static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +// callback after read nbytes from socket static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +// callback after write data to socket static void clientWriteCb(uv_write_t* req, int status); +// callback after conn to server static void clientConnCb(uv_connect_t* req, int status); static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); static void clientConnDestroy(SCliConn* pConn); +static void clientMsgDestroy(SCliMsg* pMsg); + static void* clientThread(void* arg); +static void clientProcessData(SCliConn* conn) { + STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; + SRpcInfo* pRpc = pCtx->ahandle; + SRpcMsg rpcMsg; + + rpcMsg.pCont = conn->readBuf.buf; + rpcMsg.contLen = conn->readBuf.len; + rpcMsg.ahandle = pCtx->ahandle; + (pRpc->cfp)(NULL, &rpcMsg, NULL); + // impl +} static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +static void clientTimeoutCb(uv_timer_t* handle) { + SCliThrdObj* pThrd = handle->data; + SRpcInfo* pRpc = pThrd->shandle; + int64_t currentTime = pThrd->nextTimeout; + + SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); + while (p != NULL) { + while (!QUEUE_IS_EMPTY(&p->conn)) { + queue* h = QUEUE_HEAD(&p->conn); + SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + if (c->expireTime < currentTime) { + QUEUE_REMOVE(h); + clientConnDestroy(c); + } else { + break; + } + } + p = taosHashIterate((SHashObj*)pThrd->cache, p); + } + + pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + uv_timer_start(handle, clientTimeoutCb, pRpc->idleTime * 10, 0); +} +static void* connCacheCreate(int size) { + SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + return false; +} +static void* connCacheDestroy(void* cache) { + SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); + while (connList != NULL) { + while (!QUEUE_IS_EMPTY(&connList->conn)) { + queue* h = QUEUE_HEAD(&connList->conn); + QUEUE_REMOVE(h); + SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + clientConnDestroy(c); + } + connList = taosHashIterate((SHashObj*)cache, connList); + } + taosHashClear(cache); +} + +static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { + char key[128] = {0}; + tstrncpy(key, ip, strlen(ip)); + tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + + SHashObj* pCache = cache; + SConnList* plist = taosHashGet(pCache, key, strlen(key)); + if (plist == NULL) { + SConnList list; + plist = &list; + QUEUE_INIT(&plist->conn); + taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist)); + } + + if (QUEUE_IS_EMPTY(&plist->conn)) { + return NULL; + } + queue* h = QUEUE_HEAD(&plist->conn); + QUEUE_REMOVE(h); + return QUEUE_DATA(h, SCliConn, conn); +} +static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { + char key[128] = {0}; + tstrncpy(key, ip, strlen(ip)); + tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + + STransConnCtx* ctx = ((SCliMsg*)conn->data)->ctx; + SRpcInfo* pRpc = ctx->pRpc; + conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); + // list already create before + assert(plist != NULL); + QUEUE_PUSH(&plist->conn, &conn->conn); +} +static bool clientReadComplete(SConnBuffer* data) { + STransMsgHead head; + int32_t headLen = sizeof(head); + if (data->len >= headLen) { + memcpy((char*)&head, data->buf, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); + if (msgLen > data->len) { + data->left = msgLen - data->len; + return false; + } else { + return true; + } + } else { + return false; + } +} +static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { // impl later + static const int CAPACITY = 512; + + SCliConn* conn = handle->data; + SConnBuffer* pBuf = &conn->readBuf; + if (pBuf->cap == 0) { + pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); + pBuf->len = 0; + pBuf->cap = CAPACITY; + pBuf->left = -1; + buf->base = pBuf->buf; + buf->len = CAPACITY; + } else { + if (pBuf->len >= pBuf->cap) { + if (pBuf->left == -1) { + pBuf->cap *= 2; + pBuf->buf = realloc(pBuf->buf, pBuf->cap); + } else if (pBuf->len + pBuf->left > pBuf->cap) { + pBuf->cap = pBuf->len + pBuf->left; + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); + } + } + buf->base = pBuf->buf + pBuf->len; + buf->len = pBuf->cap - pBuf->len; + } } static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // impl later - SCliConn* conn = handle->data; + SCliConn* conn = handle->data; + SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { + pBuf->len += nread; + if (clientReadComplete(pBuf)) { + tDebug("alread read complete pack"); + clientProcessData(conn); + } else { + tDebug("read halp packet, continue to read"); + } return; } + + if (nread != UV_EOF) { + tDebug("Read error %s\n", uv_err_name(nread)); + } // uv_close((uv_handle_t*)handle, clientDestroy); } @@ -84,6 +248,7 @@ static void clientConnDestroy(SCliConn* conn) { } static void clientDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; + QUEUE_REMOVE(&conn->conn); clientConnDestroy(conn); } @@ -96,15 +261,17 @@ static void clientWriteCb(uv_write_t* req, int status) { return; } - uv_read_start((uv_stream_t*)pConn->stream, clientAllocrReadBufferCb, clientReadCb); + uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } static void clientWrite(SCliConn* pConn) { - SCliMsg* pMsg = pConn->data; - SRpcHead* pHead = rpcHeadFromCont(pMsg->context->pCont); - int msgLen = rpcMsgLenFromCont(pMsg->context->contLen); - char* msg = (char*)(pHead); + SCliMsg* pCliMsg = pConn->data; + SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + + int msgLen = transMsgLenFromCont(pMsg->contLen); + char* msg = (char*)(pHead); uv_buf_t wb = uv_buf_init(msg, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); @@ -118,77 +285,52 @@ static void clientConnCb(uv_connect_t* req, int status) { return; } - SCliMsg* pMsg = pConn->data; - SEpSet* pEpSet = &pMsg->context->epSet; - SRpcMsg rpcMsg; - // rpcMsg.ahandle = pMsg->context->ahandle; - // rpcMsg.pCont = NULL; + SCliMsg* pMsg = pConn->data; + STransConnCtx* pCtx = ((SCliMsg*)(pConn->data))->ctx; + + SRpcMsg rpcMsg; + rpcMsg.ahandle = pCtx->ahandle; - char* fqdn = pEpSet->fqdn[pEpSet->inUse]; - uint32_t port = pEpSet->port[pEpSet->inUse]; if (status != 0) { // call user fp later - tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status)); - SRpcInfo* pRpc = pMsg->context->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, pEpSet); + tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); + SRpcInfo* pRpc = pMsg->ctx->pRpc; + (pRpc->cfp)(NULL, &rpcMsg, NULL); uv_close((uv_handle_t*)req->handle, clientDestroy); return; } assert(pConn->stream == req->handle); -} - -static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { - // impl later - - return NULL; -} -static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { - // impl later + clientWrite(pConn); } static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SEpSet* pEpSet = &pMsg->context->epSet; - - char* fqdn = pEpSet->fqdn[pEpSet->inUse]; - uint32_t port = pEpSet->port[pEpSet->inUse]; - - uint64_t el = taosGetTimestampUs() - pMsg->st; + uint64_t et = taosGetTimestampUs(); + uint64_t el = et - pMsg->st; tDebug("msg tran time cost: %" PRIu64 "", el); + et = taosGetTimestampUs(); - SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); + STransConnCtx* pCtx = pMsg->ctx; + SCliConn* conn = getConnFromCache(pThrd->cache, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later conn->data = pMsg; conn->writeReq->data = conn; clientWrite(conn); - // uv_buf_t wb; - // uv_write(conn->writeReq, (uv_stream_t*)conn->stream, &wb, 1, clientWriteCb); } else { - SCliConn* conn = malloc(sizeof(SCliConn)); + SCliConn* conn = calloc(1, sizeof(SCliConn)); conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->writeReq = malloc(sizeof(uv_write_t)); + QUEUE_INIT(&conn->conn); conn->connReq.data = conn; conn->data = pMsg; + struct sockaddr_in addr; - uv_ip4_addr(fqdn, port, &addr); + uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); - - // SRpcMsg rpcMsg; - // SEpSet* pEpSet = &pMsg->context->epSet; - // SRpcInfo* pRpc = pMsg->context->pRpc; - //// rpcMsg.ahandle = pMsg->context->ahandle; - // rpcMsg.pCont = NULL; - // rpcMsg.ahandle = pMsg->context->ahandle; - // uint64_t el1 = taosGetTimestampUs() - et; - // tError("msg tran back first: time cost: %" PRIu64 "", el1); - // et = taosGetTimestampUs(); - //(pRpc->cfp)(NULL, &rpcMsg, pEpSet); - // uint64_t el2 = taosGetTimestampUs() - et; - // tError("msg tran back second: time cost: %" PRIu64 "", el2); } } static void clientAsyncCb(uv_async_t* handle) { @@ -205,7 +347,8 @@ static void clientAsyncCb(uv_async_t* handle) { while (!QUEUE_IS_EMPTY(&wq)) { queue* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); - pMsg = QUEUE_DATA(h, SCliMsg, q); + + SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); clientHandleReq(pMsg, pThrd); count++; if (count >= 2) { @@ -216,11 +359,15 @@ static void clientAsyncCb(uv_async_t* handle) { static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; + SRpcInfo* pRpc = pThrd->shandle; + pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + uv_timer_start(pThrd->pTimer, clientTimeoutCb, pRpc->idleTime * 10, 0); uv_run(pThrd->loop, UV_RUN_DEFAULT); } void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SClientObj* cli = calloc(1, sizeof(SClientObj)); + memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); @@ -236,7 +383,11 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); pThrd->cliAsync->data = pThrd; + pThrd->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pThrd->pTimer); + pThrd->shandle = shandle; + int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); if (err == 0) { tDebug("sucess to create tranport-client thread %d", i); @@ -245,22 +396,44 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, } return cli; } +static void clientMsgDestroy(SCliMsg* pMsg) { + // impl later + free(pMsg); +} +void taosCloseClient(void* arg) { + // impl later + SClientObj* cli = arg; + for (int i = 0; i < cli->numOfThreads; i++) { + SCliThrdObj* pThrd = cli->pThreadObj[i]; + pthread_join(pThrd->thread, NULL); + pthread_mutex_destroy(&pThrd->msgMtx); + free(pThrd->cliAsync); + free(pThrd->loop); + free(pThrd); + } + free(cli->pThreadObj); + free(cli); +} void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later + char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]); + uint32_t port = pEpSet->port[pEpSet->inUse]; + SRpcInfo* pRpc = (SRpcInfo*)shandle; - int len = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); + int32_t flen = 0; + if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { + // imp later + } - SRpcReqContext* pContext; - pContext = (SRpcReqContext*)((char*)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); - pContext->ahandle = pMsg->ahandle; - pContext->pRpc = (SRpcInfo*)shandle; - pContext->epSet = *pEpSet; - pContext->contLen = len; - pContext->pCont = pMsg->pCont; - pContext->msgType = pMsg->msgType; - pContext->oldInUse = pEpSet->inUse; + STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + + pCtx->pRpc = (SRpcInfo*)shandle; + pCtx->ahandle = pMsg->ahandle; + pCtx->msgType = pMsg->msgType; + pCtx->ip = strdup(ip); + pCtx->port = port; assert(pRpc->connType == TAOS_CONN_CLIENT); // atomic or not @@ -268,14 +441,15 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* if (pRpc->index++ >= pRpc->numOfThreads) { pRpc->index = 0; } - SCliMsg* msg = malloc(sizeof(SCliMsg)); - msg->context = pContext; - msg->st = taosGetTimestampUs(); + SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); + cliMsg->ctx = pCtx; + cliMsg->msg = *pMsg; + cliMsg->st = taosGetTimestampUs(); SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; pthread_mutex_lock(&thrd->msgMtx); - QUEUE_PUSH(&thrd->msg, &msg->q); + QUEUE_PUSH(&thrd->msg, &cliMsg->q); pthread_mutex_unlock(&thrd->msgMtx); uv_async_send(thrd->cliAsync); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f23cfb6e2d..617abeea39 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -30,6 +30,20 @@ int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { return ret; } +int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + int ret = -1; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; + + return ret; +} void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { T_MD5_CTX context; @@ -41,6 +55,17 @@ void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { memcpy(pAuth, context.digest, sizeof(context.digest)); } +void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + memcpy(pAuth, context.digest, sizeof(context.digest)); +} int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { SRpcHead* pHead = rpcHeadFromCont(pCont); @@ -81,6 +106,54 @@ int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { return finalLen; } +bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { + return false; + // SRpcHead* pHead = rpcHeadFromCont(pCont); + bool succ = false; + int overhead = sizeof(STransCompMsg); + if (!NEEDTO_COMPRESSS_MSG(len)) { + return succ; + } + + char* buf = malloc(len + overhead + 8); // 8 extra bytes + if (buf == NULL) { + tError("failed to allocate memory for rpc msg compression, contLen:%d", len); + *flen = len; + return succ; + } + + int32_t clen = LZ4_compress_default(msg, buf, len, len + overhead); + tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", len, clen, overhead); + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (clen > 0 && clen < len - overhead) { + STransCompMsg* pComp = (STransCompMsg*)msg; + pComp->reserved = 0; + pComp->contLen = htonl(len); + memcpy(msg + overhead, buf, clen); + + tDebug("compress rpc msg, before:%d, after:%d", len, clen); + *flen = clen + overhead; + succ = true; + } else { + *flen = len; + succ = false; + } + free(buf); + return succ; +} +bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { + // impl later + return false; + STransCompMsg* pComp = (STransCompMsg*)msg; + + int overhead = sizeof(STransCompMsg); + int clen = 0; + return false; +} + SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { int overhead = sizeof(SRpcComp); SRpcHead* pNewHead = NULL; @@ -114,4 +187,8 @@ SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { return pHead; } +void transConnCtxDestroy(STransConnCtx* ctx) { + free(ctx->ip); + free(ctx); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index bc4cc695b0..4542541043 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -16,13 +16,6 @@ #ifdef USE_UV #include "transComm.h" -typedef struct SConnBuffer { - char* buf; - int len; - int cap; - int left; -} SConnBuffer; - typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -100,31 +93,32 @@ static void* acceptThread(void* arg); void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { /* * formate of data buffer: - * |<-------SRpcReqContext------->|<------------data read from socket----------->| + * |<--------------------------data from socket------------------------------->| + * |<------STransMsgHead------->|<-------------------other data--------------->| */ static const int CAPACITY = 1024; SConn* conn = handle->data; SConnBuffer* pBuf = &conn->connBuf; if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); + pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); pBuf->len = 0; pBuf->cap = CAPACITY; pBuf->left = -1; - buf->base = pBuf->buf + RPC_RESERVE_SIZE; + buf->base = pBuf->buf; buf->len = CAPACITY; } else { if (pBuf->len >= pBuf->cap) { if (pBuf->left == -1) { pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); + pBuf->buf = realloc(pBuf->buf, pBuf->cap); } else if (pBuf->len + pBuf->left > pBuf->cap) { pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); } } - buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; + buf->base = pBuf->buf + pBuf->len; buf->len = pBuf->cap - pBuf->len; } } @@ -133,11 +127,11 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b // static bool readComplete(SConnBuffer* data) { // TODO(yihao): handle pipeline later - SRpcHead rpcHead; - int32_t headLen = sizeof(rpcHead); + STransMsgHead head; + int32_t headLen = sizeof(head); if (data->len >= headLen) { - memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + memcpy((char*)&head, data->buf, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); if (msgLen > data->len) { data->left = msgLen - data->len; return false; @@ -150,21 +144,21 @@ static bool readComplete(SConnBuffer* data) { } static void uvDoProcess(SRecvInfo* pRecv) { - SRpcHead* pHead = (SRpcHead*)pRecv->msg; - SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; - SConn* pConn = pRecv->thandle; - + // impl later + STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; + SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; + SConn* pConn = pRecv->thandle; tDump(pRecv->msg, pRecv->msgLen); - terrno = 0; - SRpcReqContext* pContest; + // SRpcReqContext* pContest; // do auth and check } static int uvAuthMsg(SConn* pConn, char* msg, int len) { - SRpcHead* pHead = (SRpcHead*)msg; - int code = 0; + STransMsgHead* pHead = (STransMsgHead*)msg; + + int code = 0; if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { // secured link, or no authentication @@ -216,15 +210,15 @@ static int uvAuthMsg(SConn* pConn, char* msg, int len) { // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { - // impl later SConn* conn = handle->data; + tDebug("%p timeout since no activity", conn); } static void uvProcessData(SConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->connBuf; - p->msg = pBuf->buf + RPC_RESERVE_SIZE; + p->msg = pBuf->buf; p->msgLen = pBuf->len; p->ip = 0; p->port = 0; @@ -233,28 +227,33 @@ static void uvProcessData(SConn* pConn) { p->chandle = NULL; // - SRpcHead* pHead = (SRpcHead*)p->msg; - assert(rpcIsReq(pHead->msgType)); + STransMsgHead* pHead = (STransMsgHead*)p->msg; + assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; - pConn->ahandle = (void*)pHead->ahandle; // auth here + // auth should not do in rpc thread - int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); - if (code != 0) { - terrno = code; - return; - } + // int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); + // if (code != 0) { + // terrno = code; + // return; + //} pHead->code = htonl(pHead->code); + int32_t dlen = 0; SRpcMsg rpcMsg; - - pHead = rpcDecompressRpcMsg(pHead); - rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); + if (transDecompressMsg(NULL, 0, NULL)) { + // add compress later + // pHead = rpcDecompressRpcMsg(pHead); + } else { + // impl later + } + rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; rpcMsg.code = pHead->code; - rpcMsg.ahandle = pConn->ahandle; + rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); @@ -265,13 +264,13 @@ static void uvProcessData(SConn* pConn) { void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // opt - SConn* ctx = cli->data; - SConnBuffer* pBuf = &ctx->connBuf; + SConn* conn = cli->data; + SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; if (readComplete(pBuf)) { tDebug("alread read complete packet"); - uvProcessData(ctx); + uvProcessData(conn); } else { tDebug("read half packet, continue to read"); } @@ -320,6 +319,9 @@ void uvWorkerAsyncCb(uv_async_t* handle) { return; } uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); + + uv_timer_stop(conn->pTimer); + uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } @@ -423,7 +425,7 @@ void* workerThread(void* arg) { uv_loop_init(pThrd->loop); // SRpcInfo* pRpc = pThrd->shandle; - uv_pipe_init(pThrd->loop, pThrd->pipe, 1); + uv_pipe_init(pThrd->loop, pThrd->pipe, 0); uv_pipe_open(pThrd->pipe, pThrd->fd); pThrd->pipe->data = pThrd; @@ -491,6 +493,7 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { @@ -522,6 +525,22 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, return srv; } +void taosCloseServer(void* arg) { + // impl later + SServerObj* srv = arg; + for (int i = 0; i < srv->numOfThreads; i++) { + SWorkThrdObj* pThrd = srv->pThreadObj[i]; + pthread_join(pThrd->thread, NULL); + free(srv->pipe[i]); + free(pThrd->loop); + free(pThrd); + } + free(srv->loop); + free(srv->pipe); + free(srv->pThreadObj); + pthread_join(srv->thread, NULL); + free(srv); +} void rpcSendResponse(const SRpcMsg* pMsg) { SConn* pConn = pMsg->handle; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 6339e58560..4ccbb60cc2 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -40,6 +40,7 @@ static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); + // tsem_post(&pInfo->rspSem); tsem_post(&pInfo->rspSem); } @@ -60,6 +61,7 @@ static void *sendRequest(void *param) { // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); } diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c new file mode 100644 index 0000000000..65170d4abb --- /dev/null +++ b/source/libs/transport/test/rsclient.c @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#include "os.h" +#include "tutil.h" +#include "tglobal.h" +#include "rpcLog.h" +#include "trpc.h" +#include "taoserror.h" + +typedef struct { + int index; + SRpcEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t *pOverSem; + pthread_t thread; + void *pRpc; +} SInfo; + + +static int tcount = 0; +static int terror = 0; + +static void *sendRequest(void *param) { + SInfo *pInfo = (SInfo *)param; + SRpcMsg rpcMsg, rspMsg; + + tDebug("thread:%d, start to send request", pInfo->index); + + while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { + pInfo->num++; + rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); + rpcMsg.contLen = pInfo->msgSize; + rpcMsg.handle = pInfo; + rpcMsg.msgType = 1; + tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + + rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg); + + // handle response + if (rspMsg.code != 0) terror++; + + tDebug("thread:%d, rspLen:%d code:%d", pInfo->index, rspMsg.contLen, rspMsg.code); + + rpcFreeCont(rspMsg.pCont); + + if ( pInfo->num % 20000 == 0 ) + tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + } + + tDebug("thread:%d, it is over", pInfo->index); + tcount++; + + return NULL; +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + SRpcEpSet epSet; + int msgSize = 128; + int numOfReqs = 0; + int appThreads = 1; + char serverIp[40] = "127.0.0.1"; + char secret[TSDB_KEY_LEN] = "mypassword"; + struct timeval systemTime; + int64_t startTime, endTime; + pthread_attr_t thattr; + + // server info + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.port[0] = 7000; + epSet.port[1] = 7000; + strcpy(epSet.fqdn[0], serverIp); + strcpy(epSet.fqdn[1], "192.168.0.1"); + + // client info + memset(&rpcInit, 0, sizeof(rpcInit)); + //rpcInit.localIp = "0.0.0.0"; + rpcInit.localPort = 0; + rpcInit.label = "APP"; + rpcInit.numOfThreads = 1; + rpcInit.sessions = 100; + rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.user = "michael"; + rpcInit.secret = secret; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.connType = TAOS_CONN_CLIENT; + + for (int i=1; iindex = i; + pInfo->epSet = epSet; + pInfo->numOfReqs = numOfReqs; + pInfo->msgSize = msgSize; + tsem_init(&pInfo->rspSem, 0, 0); + pInfo->pRpc = pRpc; + pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + pInfo++; + } + + do { + usleep(1); + } while ( tcount < appThreads); + + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime)/1000.0; // mseconds + + tInfo("it takes %.3f mseconds to send %d requests to server, error num:%d", usedTime, numOfReqs*appThreads, terror); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); + + taosCloseLog(); + + return 0; +} + + diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c new file mode 100644 index 0000000000..4c7d30900b --- /dev/null +++ b/source/libs/transport/test/uv.c @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include + +#include "task.h" +#include + +#define NUM_OF_THREAD 1 +#define TIMEOUT 10000 + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t *pipe; + uv_loop_t *loop; + uv_async_t *workerAsync; // + int fd; +} SThreadObj; + +typedef struct SServerObj { + uv_tcp_t server; + uv_loop_t *loop; + int workerIdx; + int numOfThread; + SThreadObj **pThreadObj; + uv_pipe_t **pipe; +} SServerObj; + +typedef struct SConnCtx { + uv_tcp_t *pClient; + uv_timer_t *pTimer; + uv_async_t *pWorkerAsync; + int ref; +} SConnCtx; + +void echo_write(uv_write_t *req, int status) { + if (status < 0) { + fprintf(stderr, "Write error %s\n", uv_err_name(status)); + } + printf("write data to client\n"); + free(req); +} + +void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + + SConnCtx *pConn = container_of(client, SConnCtx, pClient); + pConn->ref += 1; + printf("read data %d\n", nread, buf->base, buf->len); + if (nread > 0) { + uv_write_t *req = (uv_write_t *)malloc(sizeof(uv_write_t)); + // dispatch request to database other process thread + // just write out + uv_buf_t write_out; + write_out.base = buf->base; + write_out.len = nread; + uv_write((uv_write_t *)req, client, &write_out, 1, echo_write); + free(buf->base); + return; + } + + if (nread < 0) { + if (nread != UV_EOF) + fprintf(stderr, "Read error %s\n", uv_err_name(nread)); + uv_close((uv_handle_t *)client, NULL); + } + free(buf->base); +} + +void alloc_buffer(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { + buf->base = malloc(suggested_size); + buf->len = suggested_size; +} + +void on_new_connection(uv_stream_t *s, int status) { + if (status == -1) { + // error! + return; + } + SServerObj *pObj = container_of(s, SServerObj, server); + printf("new_connection from client\n"); + + uv_tcp_t *client = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, client); + if (uv_accept(s, (uv_stream_t *)client) == 0) { + uv_write_t *write_req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_buf_t dummy_buf = uv_buf_init("a", 1); + // despatch to worker thread + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + uv_write2(write_req, (uv_stream_t *)&(pObj->pipe[pObj->workerIdx][0]), + &dummy_buf, 1, (uv_stream_t *)client, echo_write); + } else { + uv_close((uv_handle_t *)client, NULL); + } +} +void child_on_new_connection(uv_stream_t *q, ssize_t nread, + const uv_buf_t *buf) { + printf("x child_on_new_connection \n"); + if (nread < 0) { + if (nread != UV_EOF) + fprintf(stderr, "Read error %s\n", uv_err_name(nread)); + uv_close((uv_handle_t *)q, NULL); + return; + } + SThreadObj *pObj = (SThreadObj *)container_of(q, struct SThreadObj, pipe); + + uv_pipe_t *pipe = (uv_pipe_t *)q; + if (!uv_pipe_pending_count(pipe)) { + fprintf(stderr, "No pending count\n"); + return; + } + + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SConnCtx *pConn = malloc(sizeof(SConnCtx)); + + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pObj->loop, pConn->pTimer); + + pConn->pClient = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + pConn->pWorkerAsync = pObj->workerAsync; // thread safty + uv_tcp_init(pObj->loop, pConn->pClient); + + if (uv_accept(q, (uv_stream_t *)(pConn->pClient)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t *)pConn->pClient, &fd); + fprintf(stderr, "Worker Accepted fd %d\n", fd); + uv_timer_start(pConn->pTimer, timeOutCallBack, TIMEOUT, 0); + uv_read_start((uv_stream_t *)(pConn->pClient), alloc_buffer, echo_read); + } else { + uv_timer_stop(pConn->pTimer); + free(pConn->pTimer); + uv_close((uv_handle_t *)pConn->pClient, NULL); + free(pConn->pClient); + free(pConn); + } +} + +static void workerAsyncCallback(uv_async_t *handle) { + SThreadObj *pObj = container_of(handle, SThreadObj, workerAsync); + // do nothing +} +void *worker_thread(void *arg) { + SThreadObj *pObj = (SThreadObj *)arg; + int fd = pObj->fd; + pObj->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + uv_loop_init(pObj->loop); + + uv_pipe_init(pObj->loop, pObj->pipe, 1); + uv_pipe_open(pObj->pipe, fd); + + pObj->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCallback); + uv_read_start((uv_stream_t *)pObj->pipe, alloc_buffer, + child_on_new_connection); + + uv_run(pObj->loop, UV_RUN_DEFAULT); +} +int main() { + + SServerObj *server = calloc(1, sizeof(SServerObj)); + server->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + server->numOfThread = NUM_OF_THREAD; + server->workerIdx = 0; + server->pThreadObj = + (SThreadObj **)calloc(server->numOfThread, sizeof(SThreadObj *)); + server->pipe = (uv_pipe_t **)calloc(server->numOfThread, sizeof(uv_pipe_t *)); + + uv_loop_init(server->loop); + + for (int i = 0; i < server->numOfThread; i++) { + server->pThreadObj[i] = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + server->pipe[i] = (uv_pipe_t *)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, + UV_NONBLOCK_PIPE) != 0) { + return -1; + } + uv_pipe_init(server->loop, &(server->pipe[i][0]), 1); + uv_pipe_open(&(server->pipe[i][0]), fds[1]); // init write + + server->pThreadObj[i]->fd = fds[0]; + server->pThreadObj[i]->pipe = &(server->pipe[i][1]); // init read + int err = pthread_create(&(server->pThreadObj[i]->thread), NULL, + worker_thread, (void *)(server->pThreadObj[i])); + if (err == 0) { + printf("thread %d create\n", i); + } else { + printf("thread %d create failed", i); + } + + uv_tcp_init(server->loop, &server->server); + struct sockaddr_in bind_addr; + uv_ip4_addr("0.0.0.0", 7000, &bind_addr); + uv_tcp_bind(&server->server, (const struct sockaddr *)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t *)&server->server, 128, + on_new_connection)) != 0) { + fprintf(stderr, "Listen error %s\n", uv_err_name(err)); + return 2; + } + uv_run(server->loop, UV_RUN_DEFAULT); + return 0; + } +} diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 726247d450..469da11d93 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tconfig.h" -#include "ulog.h" #include "tutil.h" +#include "ulog.h" SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; int32_t tsGlobalConfigNum = 0; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a5dd1483ec..a67d66efb0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -361,6 +361,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPING, "Task dropping") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUPLICATTED_OPERATION, "Duplicatted operation") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED, "Job already freed") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_STATUS_ERROR, "Task status error") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index b35cf60dab..0000000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Base components -add_subdirectory(os) -add_subdirectory(util) -add_subdirectory(common) - -# Library components - -# Service components -# add_subdirectory(mnode) -# add_subdirectory(vnode) -# add_subdirectory(qnode) -add_subdirectory(dnode) - diff --git a/src/connector/go b/src/connector/go deleted file mode 160000 index b8f76da4a7..0000000000 --- a/src/connector/go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin deleted file mode 160000 index 4a4d79099b..0000000000 --- a/src/connector/grafanaplugin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension deleted file mode 160000 index ce52010141..0000000000 --- a/src/connector/hivemq-tdengine-extension +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ce5201014136503d34fecbd56494b67b4961056c diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt deleted file mode 100644 index f01ccb8728..0000000000 --- a/src/cq/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) - -ADD_LIBRARY(tcq ${SRC}) -IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) -ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) -ENDIF () -ADD_SUBDIRECTORY(test) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c deleted file mode 100644 index d610ed605f..0000000000 --- a/src/cq/src/cqMain.c +++ /dev/null @@ -1,532 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - -#include -#include -#include -#include - -#include "../../../include/client/taos.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tcq.h" -#include "tdataformat.h" -#include "tglobal.h" -#include "tlog.h" -#include "tsclient.h" -#include "ttimer.h" -#include "twal.h" - -#define cFatal(...) { if (cqDebugFlag & DEBUG_FATAL) { taosPrintLog("CQ FATAL ", 255, __VA_ARGS__); }} -#define cError(...) { if (cqDebugFlag & DEBUG_ERROR) { taosPrintLog("CQ ERROR ", 255, __VA_ARGS__); }} -#define cWarn(...) { if (cqDebugFlag & DEBUG_WARN) { taosPrintLog("CQ WARN ", 255, __VA_ARGS__); }} -#define cInfo(...) { if (cqDebugFlag & DEBUG_INFO) { taosPrintLog("CQ ", 255, __VA_ARGS__); }} -#define cDebug(...) { if (cqDebugFlag & DEBUG_DEBUG) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }} -#define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }} - - -typedef struct SCqObj { - tmr_h tmrId; - int64_t rid; - uint64_t uid; - int32_t tid; // table ID - int32_t rowSize; // bytes of a row - char * dstTable; - char * sqlStr; // SQL string - STSchema * pSchema; // pointer to schema array - void * pStream; - struct SCqObj *prev; - struct SCqObj *next; - SCqContext * pContext; -} SCqObj; - -static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); -static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); - -int32_t cqObjRef = -1; -int32_t cqVnodeNum = 0; - -void cqRmFromList(SCqObj *pObj) { - //LOCK in caller - - SCqContext *pContext = pObj->pContext; - - if (pObj->prev) { - pObj->prev->next = pObj->next; - } else { - pContext->pHead = pObj->next; - } - - if (pObj->next) { - pObj->next->prev = pObj->prev; - } - -} - -static void freeSCqContext(void *handle) { - if (handle == NULL) { - return; - } - SCqContext *pContext = handle; - pthread_mutex_destroy(&pContext->mutex); - - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; - cDebug("vgId:%d, CQ is closed", pContext->vgId); - free(pContext); -} - - -void cqFree(void *handle) { - if (tsEnableStream == 0) { - return; - } - SCqObj *pObj = handle; - SCqContext *pContext = pObj->pContext; - int32_t delete = 0; - - pthread_mutex_lock(&pContext->mutex); - - // free the resources associated - if (pObj->pStream) { - taos_close_stream(pObj->pStream); - pObj->pStream = NULL; - } else { - taosTmrStop(pObj->tmrId); - pObj->tmrId = 0; - } - - cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); - tdFreeSchema(pObj->pSchema); - free(pObj->dstTable); - free(pObj->sqlStr); - free(pObj); - - pContext->cqObjNum--; - - if (pContext->cqObjNum <= 0 && pContext->delete) { - delete = 1; - } - - pthread_mutex_unlock(&pContext->mutex); - - if (delete) { - freeSCqContext(pContext); - } -} - - -void cqCreateRef() { - int32_t ref = atomic_load_32(&cqObjRef); - if (ref == -1) { - ref = taosOpenRef(4096, cqFree); - - if (atomic_val_compare_exchange_32(&cqObjRef, -1, ref) != -1) { - taosCloseRef(ref); - } - } -} - - -void *cqOpen(void *ahandle, const SCqCfg *pCfg) { - if (tsEnableStream == 0) { - return NULL; - } - SCqContext *pContext = calloc(sizeof(SCqContext), 1); - if (pContext == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - atomic_add_fetch_32(&cqVnodeNum, 1); - - cqCreateRef(); - - pContext->tmrCtrl = taosTmrInit(0, 0, 0, "CQ"); - - tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user)); - tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass)); - const char* db = pCfg->db; - for (const char* p = db; *p != 0; p++) { - if (*p == '.') { - db = p + 1; - break; - } - } - tstrncpy(pContext->db, db, sizeof(pContext->db)); - pContext->vgId = pCfg->vgId; - pContext->cqWrite = pCfg->cqWrite; - tscEmbedded = 1; - - pthread_mutex_init(&pContext->mutex, NULL); - - - cDebug("vgId:%d, CQ is opened", pContext->vgId); - - return pContext; -} - - -void cqClose(void *handle) { - if (tsEnableStream == 0) { - return; - } - SCqContext *pContext = handle; - if (handle == NULL) return; - - pContext->delete = 1; - int32_t hasCq = 0; - int32_t existLoop = 0; - - // stop all CQs - cqStop(pContext); - - int64_t rid = 0; - - while (1) { - pthread_mutex_lock(&pContext->mutex); - - SCqObj *pObj = pContext->pHead; - if (pObj) { - cqRmFromList(pObj); - - rid = pObj->rid; - - hasCq = 1; - - if (pContext->pHead == NULL) { - existLoop = 1; - } - } else { - pthread_mutex_unlock(&pContext->mutex); - break; - } - - pthread_mutex_unlock(&pContext->mutex); - - taosRemoveRef(cqObjRef, rid); - - if (existLoop) { - break; - } - } - - if (hasCq == 0) { - freeSCqContext(pContext); - } - - int32_t remainn = atomic_sub_fetch_32(&cqVnodeNum, 1); - if (remainn <= 0) { - int32_t ref = cqObjRef; - cqObjRef = -1; - taosCloseRef(ref); - } -} - -void cqStart(void *handle) { - if (tsEnableStream == 0) { - return; - } - SCqContext *pContext = handle; - if (pContext->dbConn || pContext->master) return; - - cDebug("vgId:%d, start all CQs", pContext->vgId); - pthread_mutex_lock(&pContext->mutex); - - pContext->master = 1; - - SCqObj *pObj = pContext->pHead; - while (pObj) { - cqCreateStream(pContext, pObj); - pObj = pObj->next; - } - - pthread_mutex_unlock(&pContext->mutex); -} - -void cqStop(void *handle) { - if (tsEnableStream == 0) { - return; - } - - SCqContext *pContext = handle; - cDebug("vgId:%d, stop all CQs", pContext->vgId); - if (pContext->dbConn == NULL || pContext->master == 0) return; - - pthread_mutex_lock(&pContext->mutex); - - pContext->master = 0; - SCqObj *pObj = pContext->pHead; - while (pObj) { - if (pObj->pStream) { - taos_close_stream(pObj->pStream); - pObj->pStream = NULL; - cInfo("vgId:%d, id:%d CQ:%s is closed", pContext->vgId, pObj->tid, pObj->sqlStr); - } else { - taosTmrStop(pObj->tmrId); - pObj->tmrId = 0; - } - pObj = pObj->next; - } - - if (pContext->dbConn) taos_close(pContext->dbConn); - pContext->dbConn = NULL; - - pthread_mutex_unlock(&pContext->mutex); -} - -void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema, int start) { - if (tsEnableStream == 0) { - return NULL; - } - SCqContext *pContext = handle; - int64_t rid = 0; - - pthread_mutex_lock(&pContext->mutex); - - SCqObj *pObj = pContext->pHead; - while (pObj) { - if (pObj->uid == uid) { - rid = pObj->rid; - pthread_mutex_unlock(&pContext->mutex); - return (void *)rid; - } - - pObj = pObj->next; - } - - pthread_mutex_unlock(&pContext->mutex); - - pObj = calloc(sizeof(SCqObj), 1); - if (pObj == NULL) return NULL; - - pObj->uid = uid; - pObj->tid = sid; - if (dstTable != NULL) { - pObj->dstTable = strdup(dstTable); - } - pObj->sqlStr = strdup(sqlStr); - - pObj->pSchema = tdDupSchema(pSchema); - pObj->rowSize = schemaTLen(pSchema); - - cInfo("vgId:%d, id:%d CQ:%s is created", pContext->vgId, pObj->tid, pObj->sqlStr); - - pthread_mutex_lock(&pContext->mutex); - - pObj->next = pContext->pHead; - if (pContext->pHead) pContext->pHead->prev = pObj; - pContext->pHead = pObj; - - pContext->cqObjNum++; - - pObj->rid = taosAddRef(cqObjRef, pObj); - - if(start && pContext->master) { - cqCreateStream(pContext, pObj); - } else { - pObj->pContext = pContext; - } - - rid = pObj->rid; - - pthread_mutex_unlock(&pContext->mutex); - - - return (void *)rid; -} - -void cqDrop(void *handle) { - if (tsEnableStream == 0) { - return; - } - - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)handle); - if (pObj == NULL) { - return; - } - - SCqContext *pContext = pObj->pContext; - - pthread_mutex_lock(&pContext->mutex); - - cqRmFromList(pObj); - - // free the resources associated - if (pObj->pStream) { - taos_close_stream(pObj->pStream); - pObj->pStream = NULL; - } else { - taosTmrStop(pObj->tmrId); - pObj->tmrId = 0; - } - - pthread_mutex_unlock(&pContext->mutex); - - taosRemoveRef(cqObjRef, (int64_t)handle); - taosReleaseRef(cqObjRef, (int64_t)handle); -} - -static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); - if (pObj == NULL) { - return; - } - - SCqContext* pContext = pObj->pContext; - SSqlObj* pSql = (SSqlObj*)result; - if (code == TSDB_CODE_SUCCESS) { - if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) { - taos_close(pSql->pTscObj); - } - } - - pthread_mutex_lock(&pContext->mutex); - cqCreateStream(pContext, pObj); - pthread_mutex_unlock(&pContext->mutex); - - taosReleaseRef(cqObjRef, (int64_t)param); -} - -static void cqProcessCreateTimer(void *param, void *tmrId) { - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); - if (pObj == NULL) { - return; - } - - SCqContext* pContext = pObj->pContext; - - if (pContext->dbConn == NULL) { - cDebug("vgId:%d, try connect to TDengine", pContext->vgId); - taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL); - } else { - pthread_mutex_lock(&pContext->mutex); - cqCreateStream(pContext, pObj); - pthread_mutex_unlock(&pContext->mutex); - } - - taosReleaseRef(cqObjRef, (int64_t)param); -} - -// inner implement in tscStream.c -TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* desName, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), - int64_t stime, void *param, void (*callback)(void *), void* cqhandle); - -static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { - pObj->pContext = pContext; - - if (pContext->dbConn == NULL) { - cDebug("vgId:%d, create dbConn after 1000 ms", pContext->vgId); - pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, (void *)pObj->rid, pContext->tmrCtrl); - return; - } - - pObj->tmrId = 0; - - if (pObj->pStream == NULL) { - pObj->pStream = taos_open_stream_withname(pContext->dbConn, pObj->dstTable, pObj->sqlStr, cqProcessStreamRes, \ - INT64_MIN, (void *)pObj->rid, NULL, pContext); - - // TODO the pObj->pStream may be released if error happens - if (pObj->pStream) { - pContext->num++; - cDebug("vgId:%d, id:%d CQ:%s is opened", pContext->vgId, pObj->tid, pObj->sqlStr); - } else { - cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr); - } - } -} - -static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); - if (pObj == NULL) { - return; - } - - if (tres == NULL && row == NULL) { - taos_close_stream(pObj->pStream); - - pObj->pStream = NULL; - - taosReleaseRef(cqObjRef, (int64_t)param); - - return; - } - - SCqContext *pContext = pObj->pContext; - STSchema *pSchema = pObj->pSchema; - if (pObj->pStream == NULL) { - taosReleaseRef(cqObjRef, (int64_t)param); - return; - } - - cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - - int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_DATA_HEAD_SIZE + pObj->rowSize; - char *buffer = calloc(size, 1); - - SWalHead *pHead = (SWalHead *)buffer; - SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead)); - SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); - - SMemRow trow = (SMemRow)pBlk->data; - SDataRow dataRow = (SDataRow)memRowDataBody(trow); - memRowSetType(trow, SMEM_ROW_DATA); - tdInitDataRow(dataRow, pSchema); - - for (int32_t i = 0; i < pSchema->numOfCols; i++) { - STColumn *c = pSchema->columns + i; - void *val = row[i]; - if (val == NULL) { - val = (void *)getNullValue(c->type); - } else if (c->type == TSDB_DATA_TYPE_BINARY) { - val = ((char*)val) - sizeof(VarDataLenT); - } else if (c->type == TSDB_DATA_TYPE_NCHAR) { - char buf[TSDB_MAX_NCHAR_LEN]; - int32_t len = taos_fetch_lengths(tres)[i]; - taosMbsToUcs4(val, len, buf, sizeof(buf), &len); - memcpy((char *)val + sizeof(VarDataLenT), buf, len); - varDataLen(val) = len; - } - tdAppendColVal(dataRow, val, c->type, c->offset); - } - pBlk->dataLen = htonl(memRowDataTLen(trow)); - pBlk->schemaLen = 0; - - pBlk->uid = htobe64(pObj->uid); - pBlk->tid = htonl(pObj->tid); - pBlk->numOfRows = htons(1); - pBlk->sversion = htonl(pSchema->version); - pBlk->padding = 0; - - pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowDataTLen(trow); - - pMsg->header.vgId = htonl(pContext->vgId); - pMsg->header.contLen = htonl(pHead->len); - pMsg->length = pMsg->header.contLen; - pMsg->numOfBlocks = htonl(1); - - pHead->msgType = TDMT_VND_SUBMIT; - pHead->version = 0; - - // write into vnode write queue - pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL); - free(buffer); - - taosReleaseRef(cqObjRef, (int64_t)param); -} - diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt deleted file mode 100644 index d713dd7401..0000000000 --- a/src/cq/test/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -LIST(APPEND CQTEST_SRC ./cqtest.c) -ADD_EXECUTABLE(cqtest ${CQTEST_SRC}) -TARGET_LINK_LIBRARIES(cqtest tcq taos_static) diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c deleted file mode 100644 index ee52152ad8..0000000000 --- a/src/cq/test/cqtest.c +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tglobal.h" -#include "tlog.h" -#include "tcq.h" - -int64_t ver = 0; -void *pCq = NULL; - -int writeToQueue(int32_t vgId, void *data, int type, void *pMsg) { - return 0; -} - -int main(int argc, char *argv[]) { - int num = 3; - - for (int i=1; i - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_QUERY_H -#define TDENGINE_QUERY_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* qinfo_t; - -/** - * create the qinfo object according to QueryTableMsg - * @param tsdb - * @param pQueryTableMsg - * @param qinfo - * @return - */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); - - -/** - * the main query execution function, including query on both table and multitables, - * which are decided according to the tag or table name query conditions - * - * @param qinfo - * @return - */ -bool qTableQuery(qinfo_t qinfo, uint64_t *qId); - -/** - * Retrieve the produced results information, if current query is not paused or completed, - * this function will be blocked to wait for the query execution completed or paused, - * in which case enough results have been produced already. - * - * @param qinfo - * @return - */ -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); - -/** - * - * Retrieve the actual results to fill the response message payload. - * Note that this function must be executed after qRetrieveQueryResultInfo is invoked. - * - * @param qinfo qinfo object - * @param pRsp response message - * @param contLen payload length - * @return - */ -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); - -/** - * - * @param qinfo - * @return - */ -void* qGetResultRetrieveMsg(qinfo_t qinfo); - -/** - * kill current ongoing query and free query handle automatically - * @param qinfo qhandle - * @return - */ -int32_t qKillQuery(qinfo_t qinfo); - -//kill by qid -int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount); - -bool qSolveCommitNoBlock(void* pRepo, void* pMgmt); - -int32_t qQueryCompleted(qinfo_t qinfo); - -/** - * destroy query info structure - * @param qHandle - */ -void qDestroyQueryInfo(qinfo_t qHandle); - -void* qOpenQueryMgmt(int32_t vgId); -void qQueryMgmtNotifyClosed(void* pExecutor); -void qQueryMgmtReOpen(void *pExecutor); -void qCleanupQueryMgmt(void* pExecutor); -void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); -void** qAcquireQInfo(void* pMgmt, uint64_t key); -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle); -bool checkQIdEqual(void *qHandle, uint64_t qId); -int64_t genQueryId(void); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QUERY_H diff --git a/src/inc/tcq.h b/src/inc/tcq.h deleted file mode 100644 index 71efe33011..0000000000 --- a/src/inc/tcq.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_CQ_H_ -#define _TD_CQ_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tdataformat.h" - -typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); - -typedef struct { - int32_t vgId; - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] - FCqWrite cqWrite; -} SCqCfg; - -// SCqContext -typedef struct { - int32_t vgId; - int32_t master; - int32_t num; // number of continuous streams - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_NAME_LEN]; - FCqWrite cqWrite; - struct SCqObj *pHead; - void *dbConn; - void *tmrCtrl; - pthread_mutex_t mutex; - int32_t delete; - int32_t cqObjNum; -} SCqContext; - -// the following API shall be called by vnode -void *cqOpen(void *ahandle, const SCqCfg *pCfg); -void cqClose(void *handle); - -// if vnode is master, vnode call this API to start CQ -void cqStart(void *handle); - -// if vnode is slave/unsynced, vnode shall call this API to stop CQ -void cqStop(void *handle); - -// cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema, int start); - -// cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate -void cqDrop(void *handle); - -extern int32_t cqDebugFlag; - - -#ifdef __cplusplus -} -#endif - -#endif // _TD_CQ_H_ diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h deleted file mode 100644 index 130628e799..0000000000 --- a/src/inc/tsdb.h +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_TSDB_H_ -#define _TD_TSDB_H_ - -#include -#include -#include - -#include "taosdef.h" -#include "tmsg.h" -#include "tarray.h" -#include "tdataformat.h" -#include "tname.h" -#include "hash.h" -#include "tlockfree.h" -#include "tlist.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define TSDB_VERSION_MAJOR 1 -#define TSDB_VERSION_MINOR 0 - -#define TSDB_INVALID_SUPER_TABLE_ID -1 - -#define TSDB_STATUS_COMMIT_START 1 -#define TSDB_STATUS_COMMIT_OVER 2 -#define TSDB_STATUS_COMMIT_NOBLOCK 3 //commit no block, need to be solved - -// TSDB STATE DEFINITION -#define TSDB_STATE_OK 0x0 -#define TSDB_STATE_BAD_META 0x1 -#define TSDB_STATE_BAD_DATA 0x2 - -// --------- TSDB APPLICATION HANDLE DEFINITION -typedef struct { - void *appH; - void *cqH; - int (*notifyStatus)(void *, int status, int eno); - int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema, int start); - void (*cqDropFunc)(void *handle); -} STsdbAppH; - -// --------- TSDB REPOSITORY CONFIGURATION DEFINITION -typedef struct { - int32_t tsdbId; - int32_t cacheBlockSize; - int32_t totalBlocks; - int32_t daysPerFile; // day per file sharding policy - int32_t keep; // day of data to keep - int32_t keep1; - int32_t keep2; - int32_t minRowsPerFileBlock; // minimum rows per file block - int32_t maxRowsPerFileBlock; // maximum rows per file block - int8_t precision; - int8_t compression; - int8_t update; - int8_t cacheLastRow; // 0:no cache, 1: cache last row, 2: cache last NULL column 3: 1&2 -} STsdbCfg; - -#define CACHE_NO_LAST(c) ((c)->cacheLastRow == 0) -#define CACHE_LAST_ROW(c) (((c)->cacheLastRow & 1) > 0) -#define CACHE_LAST_NULL_COLUMN(c) (((c)->cacheLastRow & 2) > 0) - -// --------- TSDB REPOSITORY USAGE STATISTICS -typedef struct { - int64_t totalStorage; // total bytes occupie - int64_t compStorage; - int64_t pointsWritten; // total data points written -} STsdbStat; - -typedef struct STsdbRepo STsdbRepo; - -STsdbCfg *tsdbGetCfg(const STsdbRepo *repo); - -// --------- TSDB REPOSITORY DEFINITION -int32_t tsdbCreateRepo(int repoid); -int32_t tsdbDropRepo(int repoid); -STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); -int tsdbCloseRepo(STsdbRepo *repo, int toCommit); -int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg); -int tsdbGetState(STsdbRepo *repo); -int8_t tsdbGetCompactState(STsdbRepo *repo); -// --------- TSDB TABLE DEFINITION -typedef struct { - uint64_t uid; // the unique table ID - int32_t tid; // the table ID in the repository. -} STableId; - -// --------- TSDB TABLE configuration -typedef struct { - ETableType type; - char * name; - STableId tableId; - int32_t sversion; - char * sname; // super table name - uint64_t superUid; - STSchema * schema; - STSchema * tagSchema; - SKVRow tagValues; - char * sql; -} STableCfg; - -void tsdbClearTableCfg(STableCfg *config); - -void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes); -char *tsdbGetTableName(void *pTable); - -#define TSDB_TABLEID(_table) ((STableId*) (_table)) -#define TSDB_PREV_ROW 0x1 -#define TSDB_NEXT_ROW 0x2 - -STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); - -int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg); -int tsdbDropTable(STsdbRepo *pRepo, STableId tableId); -int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg); - -uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); - -// the TSDB repository info -typedef struct STsdbRepoInfo { - STsdbCfg tsdbCfg; - uint64_t version; // version of the repository - int64_t tsdbTotalDataSize; // the original inserted data size - int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository - // TODO: Other informations to add -} STsdbRepoInfo; -STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo); - -// the meter information report structure -typedef struct { - STableCfg tableCfg; - uint64_t version; - int64_t tableTotalDataSize; // In bytes - int64_t tableTotalDiskSize; // In bytes -} STableInfo; - -// -- FOR INSERT DATA -/** - * Insert data to a table in a repository - * @param pRepo the TSDB repository handle - * @param pData the data to insert (will give a more specific description) - * - * @return the number of points inserted, -1 for failure and the error number is set - */ -int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); - -// -- FOR QUERY TIME SERIES DATA - -typedef void *TsdbQueryHandleT; // Use void to hide implementation details - -#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 -#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 -#define BLOCK_LOAD_TABLE_RR_ORDER 3 - -// query condition to build multi-table data block iterator -typedef struct STsdbQueryCond { - STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block - int32_t numOfCols; - SColumnInfo *colList; - bool loadExternalRows; // load external rows or not - int32_t type; // data block load type: -} STsdbQueryCond; - -typedef struct STableData STableData; -typedef struct { - T_REF_DECLARE() - SRWLatch latch; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - int32_t maxTables; - STableData **tData; - SList * actList; - SList * extraBuffList; - SList * bufBlockList; - int64_t pointsAdd; // TODO - int64_t storageAdd; // TODO -} SMemTable; - -typedef struct { - SMemTable* mem; - SMemTable* imem; - SMemTable mtable; - SMemTable* omem; -} SMemSnapshot; - -typedef struct SMemRef { - int32_t ref; - SMemSnapshot snapshot; -} SMemRef; - -typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t numOfCols; - int64_t uid; - int32_t tid; -} SDataBlockInfo; - -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - -typedef struct { - void *pTable; - TSKEY lastKey; -} STableKeyInfo; - -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - -#define TSDB_BLOCK_DIST_STEP_ROWS 16 -typedef struct { - uint16_t rowSize; - uint16_t numOfFiles; - uint32_t numOfTables; - uint64_t totalSize; - uint64_t totalRows; - int32_t maxRows; - int32_t minRows; - int32_t firstSeekTimeUs; - uint32_t numOfRowsInMemTable; - uint32_t numOfSmallBlocks; - SArray *dataBlockInfos; -} STableBlockDist; - -/** - * Get the data block iterator, starting from position according to the query condition - * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the - * group by condition - * @param qinfo query info handle from query processor - * @return - */ -TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, - SMemRef *pRef); - -/** - * Get the last row of the given query time window for all the tables in STableGroupInfo object. - * Note that only one data block with only row will be returned while invoking retrieve data block function for - * all tables in this group. - * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableInfo table list. - * @return - */ -TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, - SMemRef *pRef); - - -TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); - -bool isTsdbCacheLastRow(TsdbQueryHandleT* pTsdbReadHandle); - - -/** - * get the queried table object list - * @param pHandle - * @return - */ -SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); - -/** - * get the group list according to table id from client - * @param tsdb - * @param pCond - * @param groupList - * @param qinfo - * @return - */ -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, - uint64_t qId, SMemRef *pRef); - - -/** - * get num of rows in mem table - * - * @param pHandle - * @return row size - */ - -int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); - -/** - * move to next block if exists - * - * @param pTsdbReadHandle - * @return - */ -bool tsdbNextDataBlock(TsdbQueryHandleT pTsdbReadHandle); - -/** - * Get current data block information - * - * @param pTsdbReadHandle - * @param pBlockInfo - * @return - */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); - -/** - * - * Get the pre-calculated information w.r.t. current data block. - * - * In case of data block in cache, the pBlockStatis will always be NULL. - * If a block is not completed loaded from disk, the pBlockStatis will be NULL. - - * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 - * @return - */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); - -/** - * - * The query condition with primary timestamp is passed to iterator during its constructor function, - * the returned data block must be satisfied with the time window condition in any cases, - * which means the SData data block is not actually the completed disk data blocks. - * - * @param pTsdbReadHandle query handle - * @param pColumnIdList required data columns id list - * @return - */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pTsdbReadHandle, SArray *pColumnIdList); - -/** - * Get the qualified table id for a super table according to the tag query expression. - * @param stableid. super table sid - * @param pTagCond. tag query condition - */ -int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, - int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, - SColIndex *pColIndex, int32_t numOfCols); - -/** - * destroy the created table group list, which is generated by tag query - * @param pGroupList - */ -void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); - -/** - * create the table group result including only one table, used to handle the normal table query - * - * @param tsdb tsdbHandle - * @param uid table uid - * @param pGroupInfo the generated result - * @return - */ -int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); - -/** - * - * @param tsdb - * @param pTableIdList - * @param pGroupInfo - * @return - */ -int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); - -/** - * clean up the query handle - * @param queryHandle - */ -void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); - -void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond); - -void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList); - -int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo); - -/** - * get the statistics of repo usage - * @param repo. point to the tsdbrepo - * @param totalPoints. total data point written - * @param totalStorage. total bytes took by the tsdb - * @param compStorage. total bytes took by the tsdb after compressed - */ -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage); - -int tsdbInitCommitQueue(); -void tsdbDestroyCommitQueue(); -int tsdbSyncCommit(STsdbRepo *repo); -void tsdbIncCommitRef(int vgId); -void tsdbDecCommitRef(int vgId); - -// For TSDB file sync -int tsdbSyncSend(void *pRepo, SOCKET socketFd); -int tsdbSyncRecv(void *pRepo, SOCKET socketFd); - -// For TSDB Compact -int tsdbCompact(STsdbRepo *pRepo); - -// For TSDB Health Monitor - -// no problem return true -bool tsdbNoProblem(STsdbRepo* pRepo); - -#ifdef __cplusplus -} -#endif - -#endif // _TD_TSDB_H_ diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index 4febe40054..beb324321d 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -53,22 +53,22 @@ curr_dir=$(readlink -f "$(dirname "$0")") echo $curr_dir ${curr_dir}/cleanCluster.sh -r "/data" -${curr_dir}/cleanCluster.sh -r "/data2" +#${curr_dir}/cleanCluster.sh -r "/data2" if [[ "${updateSrc}" == "yes" ]]; then ${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" fi ${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000 -${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 +#${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 if [[ "${masterDnode}" == "master" ]]; then # create all dnode into cluster - taos -s "create dnode trd02 port 8000;" + #taos -s "create dnode trd02 port 8000;" taos -s "create dnode trd03 port 7000;" - taos -s "create dnode trd03 port 8000;" + #taos -s "create dnode trd03 port 8000;" taos -s "create dnode trd04 port 7000;" - taos -s "create dnode trd04 port 8000;" + #taos -s "create dnode trd04 port 8000;" fi diff --git a/tests/script/sim/db/basic1.sim b/tests/script/sim/db/basic1.sim index 33af1c5b59..21e26784bf 100644 --- a/tests/script/sim/db/basic1.sim +++ b/tests/script/sim/db/basic1.sim @@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create database -sql create database d1 +sql create database d1 vgroups 2 sql show databases if $rows != 1 then return -1 @@ -22,6 +22,21 @@ if $data03 != 0 then return -1 endi +print =============== show vgroups1 +sql use d1 +sql show vgroups +if $rows != 2 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi + +if $data10 != 3 then + return -1 +endi + print =============== drop database sql drop database d1 sql show databases @@ -30,14 +45,68 @@ if $rows != 0 then endi print =============== more databases -sql create database d2 -sql create database d3 -sql create database d4 +sql create database d2 vgroups 2 +sql create database d3 vgroups 3 +sql create database d4 vgroups 4 sql show databases if $rows != 3 then return -1 endi +print =============== show vgroups2 +sql show d2.vgroups +if $rows != 2 then + return -1 +endi + +if $data00 != 4 then + return -1 +endi + +if $data10 != 5 then + return -1 +endi + +print =============== show vgroups3 +sql show d3.vgroups +if $rows != 3 then + return -1 +endi + +if $data00 != 6 then + return -1 +endi + +if $data10 != 7 then + return -1 +endi + +if $data20 != 8 then + return -1 +endi + +print =============== show vgroups4 +sql show d4.vgroups +if $rows != 4 then + return -1 +endi + +if $data00 != 9 then + return -1 +endi + +if $data10 != 10 then + return -1 +endi + +if $data20 != 11 then + return -1 +endi + +if $data30 != 12 then + return -1 +endi + print =============== drop database sql drop database d2 sql drop database d3 @@ -50,7 +119,7 @@ if $data00 != d4 then return -1 endi -if $data02 != 2 then +if $data02 != 4 then return -1 endi @@ -58,19 +127,12 @@ if $data03 != 0 then return -1 endi -print =============== show vgroups -sql show databases - -if $rows != 1 then - return -1 -endi - +print =============== show vgroups4 again sql_error use d1 sql use d4 sql show vgroups - -if $rows != 2 then +if $rows != 4 then return -1 endi @@ -81,15 +143,17 @@ if $data00 != 1 then return -1 endi -if $data02 != 2 then +if $data02 != 4 then return -1 endi +print =============== restart + system sh/exec.sh -n dnode1 -s stop -x SIGKILL system sh/exec.sh -n dnode1 -s start +print =============== show databases sql show databases - if $rows != 1 then return -1 endi @@ -99,7 +163,27 @@ sql_error use d1 sql use d4 sql show vgroups -if $rows != 2 then +if $rows != 4 then + return -1 +endi + +print =============== create databases +sql create database d5 vgroups 5; + +print =============== show vgroups +sql use d5 +sql show vgroups +if $rows != 5 then + return -1 +endi + +sql show d4.vgroups +if $rows != 4 then + return -1 +endi + +sql show d5.vgroups +if $rows != 5 then return -1 endi diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index a688b4c2f3..eb12da2ccb 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -7,8 +7,10 @@ print ============================ dnode1 start $i = 0 $dbPrefix = db +$stPrefix = st $tbPrefix = tb $db = $dbPrefix . $i +$st = $stPrefix . $i $tb = $tbPrefix . $i print =============== step1 @@ -68,12 +70,47 @@ if $data06 != 15 then return -1 endi +return + print =============== step6 $i = $i + 1 while $i < 5 $db = $dbPrefix . $i + $st = $stPrefix . $i + $tb = $tbPrefix . $i + + print create database $db sql create database $db + + print use $db sql use $db + + print create table $st (ts timestamp, i int) tags (j int) + sql create table $st (ts timestamp, i int) tags (j int) + + print create table $tb using $st tags(1) + sql create table $tb using $st tags(1) + + sql show stables + if $rows != 1 then + return -1 + endi + + print $data00 $data01 $data02 $data03 + if $data00 != $st then + return -1 + endi + + sql show tables + if $rows != 1 then + return -1 + endi + + print $data00 $data01 $data02 $data03 + if $data00 != $tb then + return -1 + endi + $i = $i + 1 endw @@ -86,93 +123,179 @@ while $i < 5 endw print =============== step8 -$i = 0 +$i = 1 $db = $dbPrefix . $i +$st = $stPrefix . $i $tb = $tbPrefix . $i sql create database $db sql use $db -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) -return -system sh/exec.sh -n dnode1 -s stop -x SIGINT +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi sql show tables if $rows != 1 then return -1 endi +if $data00 != $tb then + return -1 +endi + print =============== step9 sql drop database $db print =============== step10 sql create database $db sql use $db + +sql show stables +if $rows != 0 then + return -1 +endi + sql show tables if $rows != 0 then return -1 endi print =============== step11 -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) + +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi + sql show tables if $rows != 1 then return -1 endi +if $data00 != $tb then + return -1 +endi + print =============== step12 sql drop database $db print =============== step13 sql create database $db sql use $db + +sql show stables +if $rows != 0 then + return -1 +endi + sql show tables if $rows != 0 then return -1 endi -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +print ============== step14 +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) + +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi + sql show tables if $rows != 1 then return -1 endi +if $data00 != $tb then + return -1 +endi + sql insert into $tb values (now+1a, 0) sql insert into $tb values (now+2a, 1) sql insert into $tb values (now+3a, 2) sql insert into $tb values (now+4a, 3) sql insert into $tb values (now+5a, 4) - -return - sql select * from $tb if $rows != 5 then return -1 endi +sql select * from $stb +if $rows != 5 then + return -1 +endi + print =============== step14 sql drop database $db print =============== step15 sql create database $db sql use $db + +sql show stables +if $rows != 0 then + return -1 +endi + sql show tables if $rows != 0 then return -1 endi print =============== step16 -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) + +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi + sql show tables if $rows != 1 then return -1 endi + +if $data00 != $tb then + return -1 +endi + +sql insert into $tb values (now+1a, 0) +sql insert into $tb values (now+2a, 1) +sql insert into $tb values (now+3a, 2) +sql insert into $tb values (now+4a, 3) +sql insert into $tb values (now+5a, 4) + sql select * from $tb -if $rows != 0 then +if $rows != 5 then + return -1 +endi + +sql select * from $stb +if $rows != 5 then return -1 endi diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/sim/dnode/basic1.sim index 49b29a4ac8..e689bb3261 100644 --- a/tests/script/sim/dnode/basic1.sim +++ b/tests/script/sim/dnode/basic1.sim @@ -78,8 +78,8 @@ if $data02 != master then return -1 endi -print =============== create table -sql create database d1; +print =============== create database +sql create database d1 vgroups 4; sql create database d2; sql show databases @@ -90,10 +90,98 @@ endi sql use d1 sql show vgroups; -if $rows != 2 then +if $rows != 4 then return -1 endi +print =============== create table +sql use d1 + +sql create table st (ts timestamp, i int) tags (j int) +sql show stables +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +sql create table c1 using st tags(1) +sql create table c2 using st tags(2) +sql create table c3 using st tags(2) +sql create table c4 using st tags(2) +sql create table c5 using st tags(2) + +sql show tables +print $data00 $data01 $data02 + +if $rows != 5 then + return -1 +endi + +print =============== insert data +sql insert into c1 values(now+1s, 1) +sql insert into c1 values(now+2s, 2) +sql insert into c1 values(now+3s, 3) +sql insert into c2 values(now+1s, 1) +sql insert into c2 values(now+2s, 2) +sql insert into c2 values(now+3s, 3) +sql insert into c3 values(now+1s, 1) +sql insert into c3 values(now+2s, 2) +sql insert into c3 values(now+3s, 3) +sql insert into c4 values(now+1s, 1) +sql insert into c4 values(now+2s, 2) +sql insert into c4 values(now+3s, 3) +sql insert into c5 values(now+1s, 1) +sql insert into c5 values(now+2s, 2) +sql insert into c5 values(now+3s, 3) + +print =============== query data +sql select * from c1 +if $rows != 3 then + return -1 +endi + +print $data00 $data01 +print $data10 $data11 +print $data20 $data11 + +if $data01 != 1 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data21 != 3 then + return -1 +endi + +sql select * from c2 +if $rows != 3 then + return -1 +endi + +sql select * from c3 +if $rows != 3 then + return -1 +endi + +sql select * from c4 +if $rows != 3 then + return -1 +endi + +sql select * from c5 +if $rows != 3 then + return -1 +endi + +sql select * from st +#if $rows != 15 then +# return -1 +#endi + print =============== drop dnode sql drop dnode 2; sql show dnodes; diff --git a/tests/script/sim/table/basic1.sim b/tests/script/sim/table/basic1.sim index c33182dbf9..f47630b737 100644 --- a/tests/script/sim/table/basic1.sim +++ b/tests/script/sim/table/basic1.sim @@ -66,6 +66,24 @@ print =============== insert data sql insert into c1 values(now+1s, 1) sql insert into c1 values(now+2s, 2) sql insert into c1 values(now+3s, 3) +sql insert into c2 values(now+1s, 1) +sql insert into c2 values(now+2s, 2) +sql insert into c2 values(now+3s, 3) +sql insert into c3 values(now+1s, 1) +sql insert into c3 values(now+2s, 2) +sql insert into c3 values(now+3s, 3) +sql insert into c4 values(now+1s, 1) +sql insert into c4 values(now+2s, 2) +sql insert into c4 values(now+3s, 3) +sql insert into c5 values(now+1s, 1) +sql insert into c5 values(now+2s, 2) +sql insert into c5 values(now+3s, 3) +sql insert into c6 values(now+1s, 1) +sql insert into c6 values(now+2s, 2) +sql insert into c6 values(now+3s, 3) +sql insert into c7 values(now+1s, 1) +sql insert into c7 values(now+2s, 2) +sql insert into c7 values(now+3s, 3) print =============== query data sql select * from c1 @@ -89,6 +107,42 @@ if $data21 != 3 then return -1 endi +sql select * from c2 +if $rows != 3 then + return -1 +endi + +sql select * from c3 +if $rows != 3 then + return -1 +endi + +sql select * from c4 +if $rows != 3 then + return -1 +endi + +sql select * from c5 +if $rows != 3 then + return -1 +endi + +sql select * from c6 +if $rows != 3 then + return -1 +endi + +sql select * from c7 +if $rows != 3 then + return -1 +endi + +print =============== query data frpm st +sql select * from st +#if $rows != 21 then +# return -1 +#endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -114,4 +168,40 @@ if $data21 != 3 then return -1 endi +sql select * from c2 +if $rows != 3 then + return -1 +endi + +sql select * from c3 +if $rows != 3 then + return -1 +endi + +sql select * from c4 +if $rows != 3 then + return -1 +endi + +sql select * from c5 +if $rows != 3 then + return -1 +endi + +sql select * from c6 +if $rows != 3 then + return -1 +endi + +sql select * from c7 +if $rows != 3 then + return -1 +endi + +print =============== query data frpm st +sql select * from st +#if $rows != 21 then +# return -1 +#endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index d387bf483b..080f1551c2 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -26,6 +26,7 @@ char dbName[32] = "db"; char stbName[64] = "st"; int32_t numOfThreads = 1; int64_t numOfTables = 200000; +int64_t startOffset = 0; int32_t createTable = 1; int32_t insertData = 0; int32_t batchNumOfTbl = 100; @@ -84,7 +85,7 @@ void createDbAndStb() { } taos_free_result(pRes); - sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName); + sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j int)", stbName); pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { @@ -181,8 +182,19 @@ void *threadFunc(void *param) { exit(1); } - // printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - // pInfo->tableEndIndex); + pError("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", + pInfo->threadIndex, + pInfo->tableBeginIndex, + pInfo->tableEndIndex); + + pInfo->tableBeginIndex += startOffset; + pInfo->tableEndIndex += startOffset; + + pError("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", + pInfo->threadIndex, + pInfo->tableBeginIndex, + pInfo->tableEndIndex); + sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); taos_free_result(pRes); @@ -210,7 +222,7 @@ void *threadFunc(void *param) { TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); + pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); } taos_free_result(pRes); int64_t endTs = taosGetTimestampUs(); @@ -296,6 +308,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); printf("%s%s\n", indent, "-n"); printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset); printf("%s%s\n", indent, "-v"); printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups); printf("%s%s\n", indent, "-a"); @@ -329,6 +343,8 @@ void parseArgument(int32_t argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTables = atoll(argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + startOffset = atoll(argv[++i]); } else if (strcmp(argv[i], "-v") == 0) { numOfVgroups = atoi(argv[++i]); } else if (strcmp(argv[i], "-a") == 0) { @@ -352,6 +368,7 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s stbName:%s %s", GREEN, stbName, NC); pPrint("%s configDir:%s %s", GREEN, configDir, NC); pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC); + pPrint("%s startOffset:%" PRId64 " %s", GREEN, startOffset, NC); pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); pPrint("%s createTable:%d %s", GREEN, createTable, NC); @@ -381,7 +398,7 @@ int32_t main(int32_t argc, char *argv[]) { createDbAndStb(); } - pPrint("%d threads are spawned to create %" PRId64 " tables", numOfThreads, numOfTables); + pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); pthread_attr_t thattr; pthread_attr_init(&thattr); @@ -407,7 +424,7 @@ int32_t main(int32_t argc, char *argv[]) { int64_t tableFrom = 0; for (int32_t i = 0; i < numOfThreads; ++i) { pInfo[i].tableBeginIndex = tableFrom; - pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1); tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; pInfo[i].minDelay = INT64_MAX;