From 12ba6ab3683fa419c38a83f51a75c1db09bfec8d Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 21 Aug 2020 12:00:04 +0800 Subject: [PATCH 01/44] [fix bug] --- packaging/tools/makeclient.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 855d0b9c27..020a19250d 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else From 0a8d42b19942aed9d7bf0b2b4350b9d6e6217489 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 21 Aug 2020 14:54:38 +0800 Subject: [PATCH 02/44] TD-1174: jdbc without host ip --- .../com/taosdata/jdbc/TSDBConnection.java | 715 ++++++++++-------- 1 file changed, 387 insertions(+), 328 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 2ce39b7ee4..d85dc9b004 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -14,6 +14,7 @@ *****************************************************************************/ package com.taosdata.jdbc; +import java.io.*; import java.sql.Array; import java.sql.Blob; import java.sql.CallableStatement; @@ -30,336 +31,394 @@ import java.sql.SQLXML; import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; -import java.util.Enumeration; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.Executor; +import static com.sun.deploy.cache.Cache.exists; + public class TSDBConnection implements Connection { - private TSDBJNIConnector connector = null; - - protected Properties props = null; - - private String catalog = null; - - private TSDBDatabaseMetaData dbMetaData = null; - - private Properties clientInfoProps = new Properties(); - - private int timeoutMilliseconds = 0; - - private String tsCharSet = ""; - - public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { - this.dbMetaData = meta; - connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), - Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), - info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), - info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); - } - - private void connect(String host, int port, String dbName, String user, String password) throws SQLException { - this.connector = new TSDBJNIConnector(); - this.connector.connect(host, port, dbName, user, password); - - try { - this.setCatalog(dbName); - } catch (SQLException e) { - e.printStackTrace(); - } - - this.dbMetaData.setConnection(this); - } - - public TSDBJNIConnector getConnection() { - return this.connector; - } - - public Statement createStatement() throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBStatement(this.connector); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - } - - public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { - if (this.connector.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - - long id = this.connector.subscribe(topic, sql, restart, 0); - if (id == 0) { - throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); - } - - return new TSDBSubscribe(this.connector, id); - } - - public PreparedStatement prepareStatement(String sql) throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBPreparedStatement(this.connector, sql); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - } - - public CallableStatement prepareCall(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String nativeSQL(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setAutoCommit(boolean autoCommit) throws SQLException { - } - - public boolean getAutoCommit() throws SQLException { - return true; - } - - public void commit() throws SQLException { - } - - public void rollback() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void close() throws SQLException { - if (this.connector != null && !this.connector.isClosed()) { - this.connector.closeConnection(); - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); - } - } - - public boolean isClosed() throws SQLException { - return this.connector.isClosed(); - } - - /** - * A connection's database is able to provide information describing its tables, - * its supported SQL grammar, its stored procedures, the capabilities of this - * connection, etc. This information is made available through a - * DatabaseMetaData object. - * - * @return a DatabaseMetaData object for this connection - * @exception SQLException - * if a database access error occurs - */ - public DatabaseMetaData getMetaData() throws SQLException { - return this.dbMetaData; - } - - /** - * This readOnly option is not supported by TDengine. However, the method is intentionally left blank here to - * support HikariCP connection. - * @param readOnly - * @throws SQLException - */ - public void setReadOnly(boolean readOnly) throws SQLException { - } - - public boolean isReadOnly() throws SQLException { - return true; - } - - public void setCatalog(String catalog) throws SQLException { - this.catalog = catalog; - } - - public String getCatalog() throws SQLException { - return this.catalog; - } - - /** - * The transaction isolation level option is not supported by TDengine. - * This method is intentionally left empty to support HikariCP connection. - * @param level - * @throws SQLException - */ - public void setTransactionIsolation(int level) throws SQLException { - } - - /** - * The transaction isolation level option is not supported by TDengine. - * @return - * @throws SQLException - */ - public int getTransactionIsolation() throws SQLException { - return Connection.TRANSACTION_NONE; - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - // left blank to support HikariCP connection - //todo: implement getWarnings according to the warning messages returned from TDengine - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - // This method is implemented in the current way to support Spark - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - return this.prepareStatement(sql); - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Map> getTypeMap() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setTypeMap(Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setHoldability(int holdability) throws SQLException { - // intentionally left empty to support druid connection pool. - } - - /** - * the transaction is not supported by TDengine, so the opened ResultSet Objects will remain open - * @return - * @throws SQLException - */ - public int getHoldability() throws SQLException { - //intentionally left empty to support HikariCP connection. - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - public Savepoint setSavepoint() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Savepoint setSavepoint(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void rollback(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - return this.prepareStatement(sql, resultSetType, resultSetConcurrency); - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob createClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob createBlob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob createNClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML createSQLXML() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isValid(int timeout) throws SQLException { - return !this.isClosed(); - } - - public void setClientInfo(String name, String value) throws SQLClientInfoException { - clientInfoProps.setProperty(name, value); - } - - public void setClientInfo(Properties properties) throws SQLClientInfoException { - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements();) { - String name = (String) enumer.nextElement(); - clientInfoProps.put(name, properties.getProperty(name)); - } - } - - public String getClientInfo(String name) throws SQLException { - return clientInfoProps.getProperty(name); - } - - public Properties getClientInfo() throws SQLException { - return clientInfoProps; - } - - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setSchema(String schema) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getSchema() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void abort(Executor executor) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - this.timeoutMilliseconds = milliseconds; - } - - public int getNetworkTimeout() throws SQLException { - return this.timeoutMilliseconds; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + private TSDBJNIConnector connector = null; + + protected Properties props = null; + + private String catalog = null; + + private TSDBDatabaseMetaData dbMetaData = null; + + private Properties clientInfoProps = new Properties(); + + private int timeoutMilliseconds = 0; + + private String tsCharSet = ""; + + public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { + this.dbMetaData = meta; + + //load taos.cfg start + File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR)); + File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0]; + List endpoints = loadConfigEndpoints(cfgFile); + if (!endpoints.isEmpty()){ + info.setProperty(TSDBDriver.PROPERTY_KEY_HOST,endpoints.get(0).split(":")[0]); + info.setProperty(TSDBDriver.PROPERTY_KEY_PORT,endpoints.get(0).split(":")[1]); + } + //load taos.cfg end + + connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), + Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), + info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), + info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); + } + + private List loadConfigEndpoints(File cfgFile){ + List endpoints = new ArrayList<>(); + try(BufferedReader reader = new BufferedReader(new FileReader(cfgFile))) { + String line = null; + while ((line = reader.readLine())!=null){ + if (line.trim().startsWith("firstEp") || line.trim().startsWith("secondEp")){ + endpoints.add(line.substring(line.indexOf('p')+1).trim()); + } + if (endpoints.size()>1) + break; + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + return endpoints; + } + + /** + * @param cfgDirPath + * @return return the config dir + * **/ + private File loadConfigDir(String cfgDirPath) { + if (cfgDirPath == null) + return loadDefaultConfigDir(); + File cfgDir = new File(cfgDirPath); + if (!cfgDir.exists()) + return loadDefaultConfigDir(); + return cfgDir; + } + + /** + * @return search the default config dir, if the config dir is not exist will return null + * */ + private File loadDefaultConfigDir(){ + File cfgDir; + File cfgDir_linux = new File("/etc/taos"); + cfgDir = cfgDir_linux.exists() ? cfgDir_linux : null; + File cfgDir_windows = new File("C:\\TDengine\\cfg"); + cfgDir = (cfgDir == null && cfgDir_windows.exists()) ? cfgDir_windows : cfgDir; + return cfgDir; + } + + private void connect(String host, int port, String dbName, String user, String password) throws SQLException { + this.connector = new TSDBJNIConnector(); + this.connector.connect(host, port, dbName, user, password); + + try { + this.setCatalog(dbName); + } catch (SQLException e) { + e.printStackTrace(); + } + + this.dbMetaData.setConnection(this); + } + + public TSDBJNIConnector getConnection() { + return this.connector; + } + + public Statement createStatement() throws SQLException { + if (!this.connector.isClosed()) { + return new TSDBStatement(this.connector); + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + + public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { + if (this.connector.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + + long id = this.connector.subscribe(topic, sql, restart, 0); + if (id == 0) { + throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); + } + + return new TSDBSubscribe(this.connector, id); + } + + public PreparedStatement prepareStatement(String sql) throws SQLException { + if (!this.connector.isClosed()) { + return new TSDBPreparedStatement(this.connector, sql); + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + + public CallableStatement prepareCall(String sql) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String nativeSQL(String sql) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setAutoCommit(boolean autoCommit) throws SQLException { + } + + public boolean getAutoCommit() throws SQLException { + return true; + } + + public void commit() throws SQLException { + } + + public void rollback() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void close() throws SQLException { + if (this.connector != null && !this.connector.isClosed()) { + this.connector.closeConnection(); + } else { + throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); + } + } + + public boolean isClosed() throws SQLException { + return this.connector.isClosed(); + } + + /** + * A connection's database is able to provide information describing its tables, + * its supported SQL grammar, its stored procedures, the capabilities of this + * connection, etc. This information is made available through a + * DatabaseMetaData object. + * + * @return a DatabaseMetaData object for this connection + * @throws SQLException if a database access error occurs + */ + public DatabaseMetaData getMetaData() throws SQLException { + return this.dbMetaData; + } + + /** + * This readOnly option is not supported by TDengine. However, the method is intentionally left blank here to + * support HikariCP connection. + * + * @param readOnly + * @throws SQLException + */ + public void setReadOnly(boolean readOnly) throws SQLException { + } + + public boolean isReadOnly() throws SQLException { + return true; + } + + public void setCatalog(String catalog) throws SQLException { + this.catalog = catalog; + } + + public String getCatalog() throws SQLException { + return this.catalog; + } + + /** + * The transaction isolation level option is not supported by TDengine. + * This method is intentionally left empty to support HikariCP connection. + * + * @param level + * @throws SQLException + */ + public void setTransactionIsolation(int level) throws SQLException { + } + + /** + * The transaction isolation level option is not supported by TDengine. + * + * @return + * @throws SQLException + */ + public int getTransactionIsolation() throws SQLException { + return Connection.TRANSACTION_NONE; + } + + public SQLWarning getWarnings() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void clearWarnings() throws SQLException { + // left blank to support HikariCP connection + //todo: implement getWarnings according to the warning messages returned from TDengine + } + + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) + throws SQLException { + // This method is implemented in the current way to support Spark + if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { + throw new SQLException(TSDBConstants.INVALID_VARIABLES); + } + + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { + throw new SQLException(TSDBConstants.INVALID_VARIABLES); + } + + return this.prepareStatement(sql); + } + + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Map> getTypeMap() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setTypeMap(Map> map) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setHoldability(int holdability) throws SQLException { + // intentionally left empty to support druid connection pool. + } + + /** + * the transaction is not supported by TDengine, so the opened ResultSet Objects will remain open + * + * @return + * @throws SQLException + */ + public int getHoldability() throws SQLException { + //intentionally left empty to support HikariCP connection. + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + public Savepoint setSavepoint() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Savepoint setSavepoint(String name) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void rollback(Savepoint savepoint) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, + int resultSetHoldability) throws SQLException { + return this.prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, + int resultSetHoldability) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Clob createClob() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Blob createBlob() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public NClob createNClob() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public SQLXML createSQLXML() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isValid(int timeout) throws SQLException { + return !this.isClosed(); + } + + public void setClientInfo(String name, String value) throws SQLClientInfoException { + clientInfoProps.setProperty(name, value); + } + + public void setClientInfo(Properties properties) throws SQLClientInfoException { + for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { + String name = (String) enumer.nextElement(); + clientInfoProps.put(name, properties.getProperty(name)); + } + } + + public String getClientInfo(String name) throws SQLException { + return clientInfoProps.getProperty(name); + } + + public Properties getClientInfo() throws SQLException { + return clientInfoProps; + } + + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setSchema(String schema) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String getSchema() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void abort(Executor executor) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + this.timeoutMilliseconds = milliseconds; + } + + public int getNetworkTimeout() throws SQLException { + return this.timeoutMilliseconds; + } + + public T unwrap(Class iface) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isWrapperFor(Class iface) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } } From 542e450f0a1704e3672a184fa1147cc22218f207 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 21 Aug 2020 08:45:26 +0000 Subject: [PATCH 03/44] fix two case of 'unexpected generic error in RPC' log an error message if failed to call `gethostname`. don't update epset if peer returns empty epset. --- src/rpc/src/rpcMain.c | 11 +++++++---- src/util/src/tsocket.c | 21 ++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 777e4f8240..348f4bda9a 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1114,10 +1114,13 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); - tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps); - for (int i=0; iepSet.numOfEps; ++i) - pContext->epSet.port[i] = htons(pContext->epSet.port[i]); + SRpcEpSet *pEpSet = (SRpcEpSet*)pHead->content; + if (pEpSet->numOfEps > 0) { + memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); + tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps); + for (int i=0; iepSet.numOfEps; ++i) + pContext->epSet.port[i] = htons(pContext->epSet.port[i]); + } rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY) { diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index f4778f50ce..61896a86df 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -19,26 +19,25 @@ #include "tutil.h" int taosGetFqdn(char *fqdn) { - int code = 0; char hostname[1024]; hostname[1023] = '\0'; - gethostname(hostname, 1023); + if (gethostname(hostname, 1023) == -1) { + uError("failed to get hostname, reason:%s", strerror(errno)); + return -1; + } struct addrinfo hints = {0}; struct addrinfo *result = NULL; - hints.ai_flags = AI_CANONNAME; - - int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); - if (result) { - strcpy(fqdn, result->ai_canonname); - freeaddrinfo(result); - } else { + int ret = getaddrinfo(hostname, NULL, &hints, &result); + if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); - code = -1; + return -1; } - return code; + strcpy(fqdn, result->ai_canonname); + freeaddrinfo(result); + return 0; } uint32_t taosGetIpFromFqdn(const char *fqdn) { From 23948f67a74e6034ce22d7d534e7ac3a651958a0 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 21 Aug 2020 09:32:00 +0000 Subject: [PATCH 04/44] fix td-1189 --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 777e4f8240..85973f6600 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -538,7 +538,7 @@ void rpcCancelRequest(void *handle) { // signature is used to check if pContext is freed. // pContext may have been released just before app calls the rpcCancelRequest - if (pContext->signature != pContext) return; + if (pContext == NULL || pContext->signature != pContext) return; if (pContext->pConn) { tDebug("%s, app tries to cancel request", pContext->pConn->info); From 22248e87c2617c5efda40f84f79919385e0044a9 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 21 Aug 2020 20:02:18 +0800 Subject: [PATCH 05/44] TD-1174_feature: jdbc without host addr --- .../jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index d85dc9b004..86d179eae4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -34,8 +34,6 @@ import java.sql.Struct; import java.util.*; import java.util.concurrent.Executor; -import static com.sun.deploy.cache.Cache.exists; - public class TSDBConnection implements Connection { private TSDBJNIConnector connector = null; From f17904627bb7564195a35c4b5344bb2d8f2eba15 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Fri, 21 Aug 2020 21:46:46 +0800 Subject: [PATCH 06/44] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index a4111e78fc..b5f20319f7 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -25,8 +25,8 @@ 1. 确保客户端与服务端版本号是完全一致的,开源社区版和企业版也不能混用 2. 在服务器,执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行,启动*taosd* -3. 确认客户端连接时指定了正确的服务器IP地址 -4. ping服务器IP,如果没有反应,请检查你的网络 +3. 确认客户端连接时指定了正确的服务器Fully Qualified Domain Name(FQDN,通常情况下是服务器的hostname) +4. ping服务器FQDN,如果没有反应,请检查你的网络,DNS设置,hosts 文件 5. 检查防火墙设置,确认TCP/UDP 端口6030-6039 是打开的 6. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 7. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) From dc9172aef9f64c399777e102e9f4bfe5f5b64824 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Fri, 21 Aug 2020 22:19:39 +0800 Subject: [PATCH 07/44] Update insert-ch.md --- documentation20/webdocs/markdowndocs/insert-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index 5eef99e891..5e4532dfd0 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -24,6 +24,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, - 要提高写入效率,需要批量写入。一批写入的记录条数越多,插入效率就越高。但一条记录不能超过16K,一条SQL语句总长度不能超过64K(可通过参数maxSQLLength配置,最大可配置为8M)。 - TDengine支持多线程同时写入,要进一步提高写入速度,一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后,无法再提高,甚至还会下降,因为线程切频繁切换,带来额外开销。 +- 对同一张表,如果新插入记录的时间戳已经存在,新记录将被直接抛弃,也就是说,在一张表里,时间戳必须是唯一的。如果应用自动生成记录,很有可能生成的时间戳是一样的,这样,成功插入的记录条数会小于应用插入的记录条数。 ## Prometheus直接写入 [Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目,在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma),只需在Prometheus做简单配置,无需任何代码,就可将Prometheus采集的数据直接写入TDengine,并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例,可以参考。 From 30c7d790e0de4e80687fff920dc4742fb6e56f3e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 22 Aug 2020 08:06:48 +0800 Subject: [PATCH 08/44] Update Model-ch.md --- documentation20/webdocs/markdowndocs/Model-ch.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index f5776ce6ce..039fc29001 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -9,9 +9,9 @@ TDengine采用关系型数据模型,需要建库、建表。因此对于一个 不同类型的数据采集点往往具有不同的数据特征,包括数据采集频率的高低,数据保留时间的长短,副本的数目,数据块的大小等等。为让各种场景下TDengine都能最大效率的工作,TDengine建议将不同数据特征的表创建在不同的库里,因为每个库可以配置不同的存储策略。创建一个库时,除SQL标准的选项外,应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如: ```cmd -CREATE DATABASE power KEEP 365 DAYS 10 REPLICA 3 BLOCKS 4; +CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4; ``` -上述语句将创建一个名为power的库,这个库的数据将保留365天(超过365天将被自动删除),每10天一个数据文件,副本数为3, 内存块数为4。详细的语法及参数请见TAOS SQL +上述语句将创建一个名为power的库,这个库的数据将保留365天(超过365天将被自动删除),每10天一个数据文件,内存块数为4。详细的语法及参数请见TAOS SQL 创建库之后,需要使用SQL命令USE将当前库切换过来,例如: @@ -27,13 +27,15 @@ USE power; - 处于两个不同库的表是不能进行JOIN操作的。 ## 创建超级表 -一个物联网系统,往往存在多种类型的设备,比如对于电网,存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合,使用TDengine, 需要对每个类型的设备创建一超级表。以表一中的智能电表为例,可以使用如下的SQL命令创建超级表: +一个物联网系统,往往存在多种类型的设备,比如对于电网,存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合,使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例,可以使用如下的SQL命令创建超级表: ```cmd CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` 与创建普通表一样,创建表时,需要提供表名(示例中为meters),表结构Schema,即数据列的定义,为采集的物理量(示例中为ts, current, voltage, phase),数据类型可以为整型、浮点型、字符串等。除此之外,还需要提供标签的schema (示例中为location, groupId),标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签,比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 -每一种类型的数据采集点需要建立一个超级表,因此一个物联网系统,往往会有多个超级表。一个系统可以有多个DB,一个DB里可以有一到多个超级表。 +每一种类型的数据采集点需要建立一个超级表,因此一个物联网系统,往往会有多个超级表。对于电网,我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中,一个设备就可能有多个数据采集点(比如一台风力发电的风机,有的采集点采集电流、电压等电参数,有的采集点采集温度、湿度、风向等环境参数),这个时候,对这一类型的设备,需要建立多张超级表。一张超级表里定义的采集物理量必须是同时采集的。 + +一个系统可以有多个DB,一个DB里可以有一到多个超级表。 ## 创建表 TDengine对每个数据采集点需要独立建表。与标准的关系型数据一样,一张表有表名,Schema,但除此之外,还可以带有一到多个标签。创建时,需要使用超级表做模板,同时指定标签的具体值。以表一中的智能电表为例,可以使用如下的SQL命令建表: @@ -51,5 +53,5 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 ``` 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建,则使用超级表meters做模板自动创建,同时打上标签值“Beijing.Chaoyang", 2。 -**多列模型**:TDengine支持多列模型,只要这些物理量是同时采集的,这些量就可以作为不同列放在同一张表里。有的数据采集点有多组采集量,每一组的数据采集时间是不一样的,这时需要对同一个采集点建多张表。但还有一种极限的设计,单列模型,无论是否同时采集,每个采集的物理量单独建表。TDengine建议,只要采集时间一致,就采用多列模型,因为插入效率以及存储效率更高。TDengine支持最大的列数为1024列。 +**多列模型**:TDengine支持多列模型,只要物理量是同一个数据采集点同时采集的,这些量就可以作为不同列放在同一张表里。但还有一种极限的设计,单列模型,无论是否同时采集,每个采集的物理量单独建表。TDengine建议,对于同一个数据采集点,只要采集时间一样,就采用多列模型,因为插入效率以及存储效率更高。TDengine支持最大的列数为1024列。 From 0d806426cd2e57a87a35f1a5c19dff01c9d9f916 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 22 Aug 2020 08:26:48 +0800 Subject: [PATCH 09/44] Update Model-ch.md --- documentation20/webdocs/markdowndocs/Model-ch.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index 039fc29001..a7a5e14726 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -35,7 +35,7 @@ CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS 每一种类型的数据采集点需要建立一个超级表,因此一个物联网系统,往往会有多个超级表。对于电网,我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中,一个设备就可能有多个数据采集点(比如一台风力发电的风机,有的采集点采集电流、电压等电参数,有的采集点采集温度、湿度、风向等环境参数),这个时候,对这一类型的设备,需要建立多张超级表。一张超级表里定义的采集物理量必须是同时采集的。 -一个系统可以有多个DB,一个DB里可以有一到多个超级表。 +一张超级表最多容许1024列,如果一个采集点采集的物理量个数超过1024,需要建多张超级表来处理。一个系统可以有多个DB,一个DB里可以有一到多个超级表。 ## 创建表 TDengine对每个数据采集点需要独立建表。与标准的关系型数据一样,一张表有表名,Schema,但除此之外,还可以带有一到多个标签。创建时,需要使用超级表做模板,同时指定标签的具体值。以表一中的智能电表为例,可以使用如下的SQL命令建表: @@ -53,5 +53,7 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 ``` 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建,则使用超级表meters做模板自动创建,同时打上标签值“Beijing.Chaoyang", 2。 -**多列模型**:TDengine支持多列模型,只要物理量是同一个数据采集点同时采集的,这些量就可以作为不同列放在同一张表里。但还有一种极限的设计,单列模型,无论是否同时采集,每个采集的物理量单独建表。TDengine建议,对于同一个数据采集点,只要采集时间一样,就采用多列模型,因为插入效率以及存储效率更高。TDengine支持最大的列数为1024列。 +## 多列模型 vs 单列模型 +TDengine支持多列模型,只要物理量是一个数据采集点同时采集的,这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计,单列模型,每个采集的物理量都单独建表,因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位,就建三张超级表。 +TDengine建议尽可能采用多列模型,因为插入效率以及存储效率更高。但对于有些场景,一个采集点的采集量的种类经常变化,这个时候,如果采用多列模型,就需要频繁修改超级表的结构定义,让应用变的复杂,这个时候,采用单列模型就会显得简单。 From c1c8d1e3f1bbf67fc130a0dce74856c9e5dbadd5 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 22 Aug 2020 08:34:38 +0800 Subject: [PATCH 10/44] Update Model-ch.md --- documentation20/webdocs/markdowndocs/Model-ch.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index a7a5e14726..31eb6b3744 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -31,9 +31,9 @@ USE power; ```cmd CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` -与创建普通表一样,创建表时,需要提供表名(示例中为meters),表结构Schema,即数据列的定义,为采集的物理量(示例中为ts, current, voltage, phase),数据类型可以为整型、浮点型、字符串等。除此之外,还需要提供标签的schema (示例中为location, groupId),标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签,比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 +与创建普通表一样,创建表时,需要提供表名(示例中为meters),表结构Schema,即数据列的定义。第一列必须为时间戳(示例中为ts),其他列为采集的物理量(示例中为current, voltage, phase),数据类型可以为整型、浮点型、字符串等。除此之外,还需要提供标签的schema (示例中为location, groupId),标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签,比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 -每一种类型的数据采集点需要建立一个超级表,因此一个物联网系统,往往会有多个超级表。对于电网,我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中,一个设备就可能有多个数据采集点(比如一台风力发电的风机,有的采集点采集电流、电压等电参数,有的采集点采集温度、湿度、风向等环境参数),这个时候,对这一类型的设备,需要建立多张超级表。一张超级表里定义的采集物理量必须是同时采集的。 +每一种类型的数据采集点需要建立一个超级表,因此一个物联网系统,往往会有多个超级表。对于电网,我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中,一个设备就可能有多个数据采集点(比如一台风力发电的风机,有的采集点采集电流、电压等电参数,有的采集点采集温度、湿度、风向等环境参数),这个时候,对这一类型的设备,需要建立多张超级表。一张超级表里包含的采集物理量必须是同时采集的(时间戳是一致的)。 一张超级表最多容许1024列,如果一个采集点采集的物理量个数超过1024,需要建多张超级表来处理。一个系统可以有多个DB,一个DB里可以有一到多个超级表。 @@ -54,6 +54,6 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建,则使用超级表meters做模板自动创建,同时打上标签值“Beijing.Chaoyang", 2。 ## 多列模型 vs 单列模型 -TDengine支持多列模型,只要物理量是一个数据采集点同时采集的,这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计,单列模型,每个采集的物理量都单独建表,因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位,就建三张超级表。 +TDengine支持多列模型,只要物理量是一个数据采集点同时采集的(时间戳一致),这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计,单列模型,每个采集的物理量都单独建表,因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位,就建三张超级表。 TDengine建议尽可能采用多列模型,因为插入效率以及存储效率更高。但对于有些场景,一个采集点的采集量的种类经常变化,这个时候,如果采用多列模型,就需要频繁修改超级表的结构定义,让应用变的复杂,这个时候,采用单列模型就会显得简单。 From 5682851c4ebbdf30d98db1e9a1854ef874fb721d Mon Sep 17 00:00:00 2001 From: Hui Li Date: Sat, 22 Aug 2020 15:46:22 +0800 Subject: [PATCH 11/44] [TD-1048] --- cmake/define.inc | 5 ++ cmake/platform.inc | 18 ++++ packaging/deb/tarbitratord | 8 +- packaging/release.sh | 21 ++--- packaging/rpm/tarbitratord | 4 +- packaging/tools/install.sh | 31 +++++-- packaging/tools/remove.sh | 32 +++++-- packaging/tools/remove_client.sh | 2 +- src/client/src/tscFunctionImpl.c | 9 +- src/client/src/tscSubquery.c | 2 + src/os/inc/os.h | 4 + src/os/inc/osNingsi.h | 138 +++++++++++++++++++++++++++++++ src/os/src/linux/ningsi.c | 56 +++++++++++++ 13 files changed, 292 insertions(+), 38 deletions(-) create mode 100644 src/os/inc/osNingsi.h create mode 100644 src/os/src/linux/ningsi.c diff --git a/cmake/define.inc b/cmake/define.inc index c72995159f..a6e53a2597 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -77,6 +77,11 @@ IF (TD_LINUX) ADD_DEFINITIONS(-D_LINUX) ADD_DEFINITIONS(-D_TD_LINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + + IF (TD_NINGSI_60) + ADD_DEFINITIONS(-D_TD_NINGSI_60_) + MESSAGE(STATUS "set ningsi macro to true") + ENDIF () SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") diff --git a/cmake/platform.inc b/cmake/platform.inc index 71eee9d507..7834a35411 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -22,6 +22,9 @@ SET(TD_LINUX FALSE) SET(TD_MIPS_64 FALSE) SET(TD_MIPS_32 FALSE) SET(TD_APLHINE FALSE) + SET(TD_NINGSI FALSE) + SET(TD_NINGSI_60 FALSE) + SET(TD_NINGSI_80 FALSE) SET(TD_WINDOWS FALSE) SET(TD_WINDOWS_64 FALSE) SET(TD_WINDOWS_32 FALSE) @@ -99,3 +102,18 @@ ELSEIF (${CPUTYPE} MATCHES "x86") ELSE () MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () + +# cmake -DOSTYPE=Ningsi +IF (${OSTYPE} MATCHES "Ningsi60") + SET(TD_NINGSI TRUE) + SET(TD_NINGSI_60 TRUE) + MESSAGE(STATUS "input osType: Ningsi60") +ELSEIF (${OSTYPE} MATCHES "Ningsi80") + SET(TD_NINGSI TRUE) + SET(TD_NINGSI_80 TRUE) + MESSAGE(STATUS "input osType: Ningsi80") +ELSEIF (${OSTYPE} MATCHES "Linux") + MESSAGE(STATUS "input osType: Linux") +ELSE () + MESSAGE(STATUS "input osType unknown: " ${OSTYPE}) +ENDIF () \ No newline at end of file diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord index bf4dba7258..3f97c3c0c2 100644 --- a/packaging/deb/tarbitratord +++ b/packaging/deb/tarbitratord @@ -7,19 +7,19 @@ # chkconfig: 2345 99 01 # ### BEGIN INIT INFO -# Provides: TDEngine +# Provides: taoscluster # Required-Start: $local_fs $network $syslog # Required-Stop: $local_fs $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: Starts TDEngine tarbitrator -# Description: Starts TDEngine tarbitrator, a arbitrator +# Short-Description: Starts taoscluster tarbitrator +# Description: Starts taoscluster tarbitrator, a arbitrator ### END INIT INFO set -e PATH="/bin:/usr/bin:/sbin:/usr/sbin" -NAME="tarbitrator" +NAME="taoscluster" USER="root" GROUP="root" DAEMON="/usr/local/taos/bin/tarbitrator" diff --git a/packaging/release.sh b/packaging/release.sh index dceb879126..3ba1ba72b2 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -7,22 +7,20 @@ set -e # releash.sh -v [cluster | edge] # -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] -# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | ...] +# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] # -l [full | lite] -# -s [static | dynamic] # -n [2.0.0.3] # set parameters by default value verMode=edge # [cluster, edge] verType=stable # [stable, beta] cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] -osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | ...] +osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] -soMode=dynamic # [static | dynamic] verNumber="" -while getopts "hv:V:c:o:l:s:n:" arg +while getopts "hv:V:c:o:l:n:" arg do case $arg in v) @@ -41,10 +39,6 @@ do #echo "pagMode=$OPTARG" pagMode=$(echo $OPTARG) ;; - s) - #echo "soMode=$OPTARG" - soMode=$(echo $OPTARG) - ;; n) #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) @@ -56,10 +50,9 @@ do h) echo "Usage: `basename $0` -v [cluster | edge] " echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] " - echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | ...] " + echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " echo " -V [stable | beta] " echo " -l [full | lite] " - echo " -s [static | dynamic] " echo " -n [version number] " exit 0 ;; @@ -70,7 +63,7 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} verNumber=${verNumber}" curr_dir=$(pwd) @@ -230,9 +223,9 @@ cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DSOMODE=${soMode} + cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} else - cmake ../../ -DCPUTYPE=${cpuType} -DSOMODE=${soMode} + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} fi else echo "input cpuType=${cpuType} error!!!" diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord index 5d3f2b9c79..68138f5c1d 100644 --- a/packaging/rpm/tarbitratord +++ b/packaging/rpm/tarbitratord @@ -7,10 +7,10 @@ # # ### BEGIN INIT INFO -# Provides: TDEngine +# Provides: taoscluster # Required-Start: $network $local_fs $remote_fs # Required-Stop: $network $local_fs $remote_fs -# Short-Description: start and stop taosd +# Short-Description: start and stop tarbitrator # Description: tarbitrator is a arbitrator used in TDengine cluster. ### END INIT INFO diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 6f54993f91..64de09df6d 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -76,7 +76,11 @@ fi # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) +if [[ -d /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi #echo "osinfo: ${osinfo}" os_type=0 if echo $osinfo | grep -qwi "ubuntu" ; then @@ -95,8 +99,10 @@ elif echo $osinfo | grep -qwi "fedora" ; then # echo "This is fedora system" os_type=2 else - echo "${osinfo}: This is an officially unverified linux system, If there are any problems with the installation and operation, " - echo "please feel free to contact taosdata.com for support." + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." os_type=1 fi @@ -192,13 +198,12 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : #${csudo} rm -rf ${v15_java_app_dir} || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - if [ -d ${lib64_link_dir} ]; then + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi @@ -306,14 +311,27 @@ function clean_service_on_sysvinit() { fi if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} chkconfig --del taosd || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : + fi elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} insserv -r taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} insserv -r tarbitratord || : + fi elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} update-rc.d -f taosd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : + fi fi ${csudo} rm -f ${service_config_dir}/taosd || : @@ -326,7 +344,6 @@ function clean_service_on_sysvinit() { function install_service_on_sysvinit() { clean_service_on_sysvinit - sleep 1 # Install taosd service @@ -446,7 +463,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo >> ${tarbitratord_service_config}" ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - # ${csudo} systemctl enable tarbitratord + #${csudo} systemctl enable tarbitratord nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 63e09dc568..c775f98b83 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -72,7 +72,6 @@ function clean_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : } @@ -86,7 +85,7 @@ function clean_lib() { function clean_header() { # Remove link ${csudo} rm -f ${inc_link_dir}/taos.h || : - ${csudo} rm -f ${inc_link_dir}/taoserror.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : } function clean_config() { @@ -148,15 +147,27 @@ function clean_service_on_sysvinit() { ${csudo} service tarbitratord stop || : fi - if ((${initd_mod}==1)); then + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} chkconfig --del taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : - elif ((${initd_mod}==2)); then + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} insserv -r taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} insserv -r tarbitratord || : - elif ((${initd_mod}==3)); then + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} update-rc.d -f taosd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : + fi fi ${csudo} rm -f ${service_config_dir}/taosd || : @@ -196,13 +207,20 @@ ${csudo} rm -rf ${data_link_dir} || : ${csudo} rm -rf ${install_main_dir} ${csudo} rm -rf ${install_nginxd_dir} +if [[ -d /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi -osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) if echo $osinfo | grep -qwi "ubuntu" ; then # echo "this is ubuntu system" ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : +elif echo $osinfo | grep -qwi "debian" ; then +# echo "this is debian system" + ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : elif echo $osinfo | grep -qwi "centos" ; then - echo "this is centos system" +# echo "this is centos system" ${csudo} rpm -e --noscripts tdengine || : fi diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 4bc278fcf0..7cbf524d04 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -37,7 +37,7 @@ function kill_client() { function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosump || : + ${csudo} rm -f ${bin_link_dir}/taosdemo || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 5c708dffee..04c6ef47a5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -56,7 +56,8 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ - __ctx->tag = (tVariant){.i64Key = (ts), .nType = TSDB_DATA_TYPE_BIGINT}; \ + __ctx->tag.i64Key = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ } \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ @@ -963,7 +964,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { - __ctx->tag = (tVariant){.i64Key = key, .nType = TSDB_DATA_TYPE_BIGINT}; + __ctx->tag.i64Key = key; + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; } aAggs[TSDB_FUNC_TAG].xFunction(__ctx); @@ -1867,7 +1869,8 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; if (ctx->functionId == TSDB_FUNC_TS_DUMMY) { - ctx->tag = (tVariant) {.nType = TSDB_DATA_TYPE_BIGINT, .i64Key = tsKey}; + ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; + ctx->tag.i64Key = tsKey; } tVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 75644c355c..9125ec84ea 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _GNU_SOURCE + #include "os.h" #include "qAst.h" diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 4953416bde..df4b847fbb 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -40,6 +40,10 @@ extern "C" { #include "osAlpine.h" #endif +#ifdef _TD_NINGSI_60_ +#include "osNingsi.h" +#endif + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include "osWindows.h" #endif diff --git a/src/os/inc/osNingsi.h b/src/os/inc/osNingsi.h new file mode 100644 index 0000000000..da7f796b59 --- /dev/null +++ b/src/os/inc/osNingsi.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_NINGSI_H +#define TDENGINE_OS_NINGSI_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_OS_FUNC_ATOMIC +/* + * type __sync_fetch_and_add (type *ptr, type value); + * type __sync_fetch_and_sub (type *ptr, type value); + * type __sync_fetch_and_or (type *ptr, type value); + * type __sync_fetch_and_and (type *ptr, type value); + * type __sync_fetch_and_xor (type *ptr, type value); + * type __sync_fetch_and_nand (type *ptr, type value); + * type __sync_add_and_fetch (type *ptr, type value); + * type __sync_sub_and_fetch (type *ptr, type value); + * type __sync_or_and_fetch (type *ptr, type value); + * type __sync_and_and_fetch (type *ptr, type value); + * type __sync_xor_and_fetch (type *ptr, type value); + * type __sync_nand_and_fetch (type *ptr, type value); + * + * bool __sync_bool_compare_and_swap (type*ptr, type oldval, type newval, ...) + * type __sync_val_compare_and_swap (type *ptr, type oldval, ?type newval, ...) + * */ + +#define atomic_load_8(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_16(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_32(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_64(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_ptr(ptr) __sync_fetch_and_add((ptr), 0) + +#define atomic_store_8(ptr, val) (*(ptr)=(val)) +#define atomic_store_16(ptr, val) (*(ptr)=(val)) +#define atomic_store_32(ptr, val) (*(ptr)=(val)) +#define atomic_store_64(ptr, val) (*(ptr)=(val)) +#define atomic_store_ptr(ptr, val) (*(ptr)=(val)) + +int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ); +int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ); +int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ); +int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ); +void* atomic_exchange_ptr_impl( void **ptr, void *val ); + +#define atomic_exchange_8(ptr, val) atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val) +#define atomic_exchange_16(ptr, val) atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val) +#define atomic_exchange_32(ptr, val) atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val) +#define atomic_exchange_64(ptr, val) atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val) +#define atomic_exchange_ptr(ptr, val) atomic_exchange_ptr_impl((void **)ptr, (void*)val) + +#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap + +#define atomic_add_fetch_8(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_16(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_32(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_64(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_ptr(ptr, val) __sync_add_and_fetch((ptr), (val)) + +#define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_16(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_32(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_64(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_ptr(ptr, val) __sync_fetch_and_add((ptr), (val)) + +#define atomic_sub_fetch_8(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_16(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_32(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_64(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_ptr(ptr, val) __sync_sub_and_fetch((ptr), (val)) + +#define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_16(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_32(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_64(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_ptr(ptr, val) __sync_fetch_and_sub((ptr), (val)) + +#define atomic_and_fetch_8(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_16(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_32(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_64(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_ptr(ptr, val) __sync_and_and_fetch((ptr), (val)) + +#define atomic_fetch_and_8(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_16(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_32(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_64(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_ptr(ptr, val) __sync_fetch_and_and((ptr), (val)) + +#define atomic_or_fetch_8(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_16(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_32(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_64(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_ptr(ptr, val) __sync_or_and_fetch((ptr), (val)) + +#define atomic_fetch_or_8(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_16(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_32(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_64(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_ptr(ptr, val) __sync_fetch_and_or((ptr), (val)) + +#define atomic_xor_fetch_8(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_16(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_32(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_64(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_ptr(ptr, val) __sync_xor_and_fetch((ptr), (val)) + +#define atomic_fetch_xor_8(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_16(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_32(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val)) + + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/src/linux/ningsi.c b/src/os/src/linux/ningsi.c new file mode 100644 index 0000000000..793ccac84a --- /dev/null +++ b/src/os/src/linux/ningsi.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifdef _TD_NINGSI_60_ +void* atomic_exchange_ptr_impl(void** ptr, void* val ) { + void *old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ) { + int8_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ) { + int16_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ) { + int32_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { + int64_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +#endif + From c984139ca1ffa72fa11dd1902ca0b261b6846aed Mon Sep 17 00:00:00 2001 From: Hui Li Date: Sat, 22 Aug 2020 16:41:09 +0800 Subject: [PATCH 12/44] [TD-1196] --- src/dnode/src/dnodeMgmt.c | 3 +++ src/inc/taosmsg.h | 1 + src/mnode/src/mnodeDnode.c | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 4b06626753..d7088a337a 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -717,6 +717,9 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); + pStatus->clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 265bf47d6d..1daa1da038 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -575,6 +575,7 @@ typedef struct { int32_t maxVgroupsPerDb; char arbitrator[TSDB_EP_LEN]; // tsArbitrator char timezone[64]; // tsTimezone + int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale char charset[TSDB_LOCALE_LEN]; // tsCharset } SClusterCfg; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 261f63d5cf..5da57a3093 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -363,10 +363,15 @@ static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { mError("\"arbitrator\"[%s - %s] cfg parameters inconsistent", clusterCfg->arbitrator, tsArbitrator); return false; } - if (0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) { - mError("\"timezone\"[%s - %s] cfg parameters inconsistent", clusterCfg->timezone, tsTimezone); + + int64_t checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { + mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64"] cfg parameters inconsistent", clusterCfg->timezone, tsTimezone, clusterCfg->checkTime, checkTime); return false; } + if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) { mError("\"locale\"[%s - %s] cfg parameters inconsistent", clusterCfg->locale, tsLocale); return false; From 467962e41eb4bfc8267873da461930d9e53466c5 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 23 Aug 2020 19:16:46 +0800 Subject: [PATCH 13/44] Update administrator-ch.md --- documentation20/webdocs/markdowndocs/administrator-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 813d06a660..97f9f9e839 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -82,7 +82,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修 - firstEp: taosd启动时,主动连接的集群中第一个dnode的end point, 默认值为localhost:6030。 - secondEp: taosd启动时,如果first连接不上,尝试连接集群中第二个dnode的end point, 默认值为空。 -- fqdn:数据节点的FQDN。如果为空,将自动获取操作系统配置的第一个, 默认值为空。 +- fqdn:数据节点的FQDN,缺省为操作系统配置的第一个hostname。如果习惯IP地址访问,可设置为该节点的IP地址。 - serverPort:taosd启动后,对外服务的端口号,默认值为6030。 - httpPort: RESTful服务使用的端口号,所有的HTTP请求(TCP)都需要向该接口发起查询/写入请求, 默认值为6041。 - dataDir: 数据文件目录,所有的数据文件都将写入该目录。默认值:/var/lib/taos。 From a5702abfc2f85a4637eb3091214a56c848fb3366 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 23 Aug 2020 19:20:05 +0800 Subject: [PATCH 14/44] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index c8c72abc5c..5c5ae5fc72 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -2,7 +2,7 @@ 多个taosd的运行实例可以组成一个集群,以保证TDengine的高可靠运行,并提供水平扩展能力。要了解TDengine 2.0的集群管理,需要对集群的基本概念有所了解,请看TDengine 2.0整体架构一章。而且在安装集群之前,请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验过单节点功能。 -集群的每个节点是由End Point来唯一标识的,End Point是由FQDN(Fully Qualified Domain Name)外加Port组成,比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname,可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号,缺省是6030,但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个,但也可以通过taos.cfg里配置参数fqdn进行指定。 +集群的每个节点是由End Point来唯一标识的,End Point是由FQDN(Fully Qualified Domain Name)外加Port组成,比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname,可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号,缺省是6030,但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个,但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问,可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单,除添加和删除节点需要人工干预之外,其他全部是自动完成,最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 From ca35b6dfcab4b51e1feb4129568886a9ef72b976 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 23 Aug 2020 22:27:34 +0800 Subject: [PATCH 15/44] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 5c5ae5fc72..31d70e73e1 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -2,7 +2,7 @@ 多个taosd的运行实例可以组成一个集群,以保证TDengine的高可靠运行,并提供水平扩展能力。要了解TDengine 2.0的集群管理,需要对集群的基本概念有所了解,请看TDengine 2.0整体架构一章。而且在安装集群之前,请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验过单节点功能。 -集群的每个节点是由End Point来唯一标识的,End Point是由FQDN(Fully Qualified Domain Name)外加Port组成,比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname,可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号,缺省是6030,但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个,但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问,可以将参数fqdn设置为本节点的IP地址。 +集群的每个节点是由End Point来唯一标识的,End Point是由FQDN(Fully Qualified Domain Name)外加Port组成,比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname,可通过Linux命令`hostname -f`获取。端口是这个节点对外服务的端口号,缺省是6030,但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个,但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问,可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单,除添加和删除节点需要人工干预之外,其他全部是自动完成,最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 @@ -16,7 +16,7 @@ TDengine的集群管理极其简单,除添加和删除节点需要人工干预 **第四步**:检查、配置所有节点的FQDN: -1. 每个节点上执行命令`hostname`,查看和确认所有节点的hostname是不相同的; +1. 每个节点上执行命令`hostname -f`,查看和确认所有节点的hostname是不相同的; 2. 每个节点上执行`ping host`, 其中host是其他节点的hostname, 看能否ping通其它节点; 如果不能ping通,需要检查网络设置, 或/etc/hosts文件,或DNS的配置。如果无法ping通,是无法组成集群的。 3. 每个节点的FQDN就是输出的hostname外加端口号,比如h1.taosdata.com:6030 From fa8dd5f4804f154f34af78c8149c4aba48634c24 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Sun, 23 Aug 2020 22:40:55 +0800 Subject: [PATCH 16/44] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index b5f20319f7..5b5da4b32a 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -1,4 +1,4 @@ -#常见问题 +# 常见问题 #### 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么?☆☆☆ @@ -25,8 +25,8 @@ 1. 确保客户端与服务端版本号是完全一致的,开源社区版和企业版也不能混用 2. 在服务器,执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行,启动*taosd* -3. 确认客户端连接时指定了正确的服务器Fully Qualified Domain Name(FQDN,通常情况下是服务器的hostname) -4. ping服务器FQDN,如果没有反应,请检查你的网络,DNS设置,hosts 文件 +3. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得) +4. ping服务器FQDN,如果没有反应,请检查你的网络,DNS设置,或客户端所在计算机的系统hosts文件 5. 检查防火墙设置,确认TCP/UDP 端口6030-6039 是打开的 6. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 7. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) From 0290613f4cdb585bf2b0e3dbc842d0a5c2591712 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Mon, 24 Aug 2020 09:21:33 +0800 Subject: [PATCH 17/44] Update faq-ch.md --- .../webdocs/markdowndocs/faq-ch.md | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 5b5da4b32a..ba2d91c4ac 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -1,6 +1,6 @@ # 常见问题 -#### 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么?☆☆☆ +## 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么?☆☆☆ 2.0版本在之前版本的基础上,进行了完全的重构,配置文件和数据文件是不兼容的。在升级之前务必进行如下操作: @@ -10,16 +10,16 @@ 4. 安装最新稳定版本的TDengine 5. 如果数据需要迁移数据或者数据文件损坏,请联系涛思数据官方技术支持团队,进行协助解决 -#### 2. Windows平台下JDBCDriver找不到动态链接库,怎么办? +## 2. Windows平台下JDBCDriver找不到动态链接库,怎么办? 请看为此问题撰写的技术博客 -#### 3. 创建数据表时提示more dnodes are needed +## 3. 创建数据表时提示more dnodes are needed 请看为此问题撰写的技术博客 -#### 4. 如何让TDengine crash时生成core文件? +## 4. 如何让TDengine crash时生成core文件? 请看为此问题撰写的技术博客 -#### 5. 遇到错误"Unable to establish connection", 我怎么办? +## 5. 遇到错误"Unable to establish connection", 我怎么办? 客户端遇到链接故障,请按照下面的步骤进行检查: @@ -36,39 +36,36 @@ 检查客户端侧TCP端口链接是否工作:`nc {hostIP} {port}` -#### 6. 虽然语法正确,为什么我还是得到 "Invalid SQL" 错误 +## 6. 虽然语法正确,为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确,2.0之前版本,请检查SQL语句长度是否超过64K。如果超过,也会返回这个错误。 -#### 7. 是否支持validation queries? +## 7. 是否支持validation queries? TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。 -#### 8. 我可以删除或更新一条记录吗? +## 8. 我可以删除或更新一条记录吗? 不能。因为TDengine是为联网设备采集的数据设计的,不容许修改。但TDengine提供数据保留策略,只要数据记录超过保留时长,就会被自动删除。 -#### 10. 我怎么创建超过250列的表? +## 10. 我怎么创建超过250列的表? 使用2.0及其以上版本,默认支持1024列;2.0之前的版本,TDengine最大允许创建250列的表。但是如果确实超过限值,建议按照数据特性,逻辑地将这个宽表分解成几个小表。 -#### 10. 最有效的写入数据的方法是什么? +## 10. 最有效的写入数据的方法是什么? 批量插入。每条写入语句可以一张表同时插入多条记录,也可以同时插入多张表的记录。 -#### 11. 最有效的写入数据的方法是什么?windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决? +## 11. 最有效的写入数据的方法是什么?windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决? windows下插入nchar类的数据中如果有中文,请先确认系统的地区设置成了中国(在Control Panel里可以设置),这时cmd中的`taos`客户端应该已经可以正常工作了;如果是在IDE里开发Java应用,比如Eclipse, Intellij,请确认IDE里的文件编码为GBK(这是Java默认的编码类型),然后在生成Connection时,初始化客户端的配置,具体语句如下: - -​ Class.forName("com.taosdata.jdbc.TSDBDriver"); - -​ Properties properties = new Properties(); - -​ properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); - -​ Connection = DriverManager.getConnection(url, properties); - -#### 12.TDengine GO windows驱动的如何编译? +```JAVA +Class.forName("com.taosdata.jdbc.TSDBDriver"); +Properties properties = new Properties(); +properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); +Connection = DriverManager.getConnection(url, properties); +``` +## 12.TDengine GO windows驱动的如何编译? 请看为此问题撰写的技术博客 From 72352f036702effabc71960f218b1bd284718eed Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 10:04:04 +0800 Subject: [PATCH 18/44] TD-1205 add error code to crash_gen.py --- src/mnode/src/mnodeDb.c | 8 ++++++-- tests/pytest/crash_gen.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 78c022ce46..26b4430777 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -41,7 +41,7 @@ static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg); +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg); static int32_t mnodeDropDb(SMnodeMsg *newMsg); static int32_t mnodeSetDbDropping(SDbObj *pDb); static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -343,7 +343,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { return code; } -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) { +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); if (code != 0) return code; @@ -393,6 +393,9 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs return code; } + pMsg->pDb = pDb; + mnodeIncDbRef(pDb); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, @@ -405,6 +408,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); + pMsg->pDb = NULL; mnodeDestroyDb(pDb); } diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 8edba3903d..7d3eb959c0 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1492,6 +1492,7 @@ class Task(): 0x386, # DB is being dropped?! 0x503, 0x510, # vnode not in ready state + 0x14, # db not ready, errno changed 0x600, 1000 # REST catch-all error ]: From 852ffa43d11d63f335a13e09fe86ca935d3d186d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 10:33:26 +0800 Subject: [PATCH 19/44] TD-1205 re-run ci --- tests/script/general/table/delete_writing.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index 868936dda7..b55d55eb8a 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -35,7 +35,7 @@ sleep 1000 print ======== step1 $x = 1 -while $x < 20 +while $x < 15 print drop table times $x sql drop table db.tb -x step1 From 5ac0d1376e1700725f12bcc7645063c05a56af22 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 07:15:02 +0000 Subject: [PATCH 20/44] TD-1212 add replica paras for taosdemo --- src/kit/taosdemo/taosdemo.c | 63 +++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 74bdf6f074..127a53db60 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -57,29 +57,30 @@ extern char configDir[]; /* Used by main to communicate with parse_opt. */ typedef struct DemoArguments { - char *host; - uint16_t port; - char *user; - char *password; - char *database; - char *tb_prefix; - char *sqlFile; - bool use_metric; - bool insert_only; - char *output_file; - int mode; - char *datatype[MAX_NUM_DATATYPE+1]; - int len_of_binary; - int num_of_CPR; - int num_of_threads; - int num_of_RPR; - int num_of_tables; - int num_of_DPT; - int abort; - int order; - int rate; - int method_of_delete; - char **arg_list; + char * host; + uint16_t port; + char * user; + char * password; + char * database; + int replica; + char * tb_prefix; + char * sqlFile; + bool use_metric; + bool insert_only; + char * output_file; + int mode; + char * datatype[MAX_NUM_DATATYPE + 1]; + int len_of_binary; + int num_of_CPR; + int num_of_threads; + int num_of_RPR; + int num_of_tables; + int num_of_DPT; + int abort; + int order; + int rate; + int method_of_delete; + char ** arg_list; } SDemoArguments; #ifdef LINUX @@ -90,6 +91,7 @@ typedef struct DemoArguments { {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, {0, 's', "sql file", 0, "The select sql file.", 3}, {0, 'M', 0, 0, "Use metric flag.", 13}, @@ -225,6 +227,13 @@ typedef struct DemoArguments { arguments->rate = 10; } break; + case 'a': + arguments->replica = atoi(arg); + if (arguments->replica > 3 || arguments->replica < 1) + { + arguments->replica = 1; + } + break; case 'D': arguments->method_of_delete = atoi(arg); if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) @@ -273,6 +282,8 @@ typedef struct DemoArguments { printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); printf("%s%s\n", indent, "-d"); printf("%s%s%s\n", indent, indent, "database, Destination database. Default is 'test'."); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s\n", indent, indent, "replica, Set the replica parameters of the database, Default 1, min: 1, max: 3."); printf("%s%s\n", indent, "-m"); printf("%s%s%s\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); printf("%s%s\n", indent, "-s"); @@ -396,6 +407,11 @@ typedef struct DemoArguments { if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) { arguments->rate = 10; } + } else if (strcmp(argv[i], "-a") == 0) { + arguments->replica = atoi(argv[++i]); + if (arguments->rate > 3 || arguments->rate < 1) { + arguments->rate = 1; + } } else if (strcmp(argv[i], "-D") == 0) { arguments->method_of_delete = atoi(argv[++i]); if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { @@ -499,6 +515,7 @@ int main(int argc, char *argv[]) { "root", // user "taosdata", // password "test", // database + 1, // replica "t", // tb_prefix NULL, false, // use_metric From 82d12eec6a9752b5a9a89a0915f3935e588161de Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 16:02:49 +0800 Subject: [PATCH 21/44] TD-1201 fix compile error in centos --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c74a35030e..9282fa74fb 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -125,7 +125,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], sizeof(pEpSet->fqdn[i])); + tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); From af20a536cdf28b94a8eab7d83251556df9c5af1e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 16:49:37 +0800 Subject: [PATCH 22/44] TD-1212 --- src/kit/taosdemo/taosdemo.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 127a53db60..6469990c78 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -581,6 +581,7 @@ int main(int argc, char *argv[]) { int count_data_type = 0; char dataString[STRING_LEN]; bool do_aggreFunc = true; + int replica = arguments.replica; if (NULL != arguments.sqlFile) { TAOS* qtaos = taos_connect(ip_addr, user, pass, db_name, port); @@ -678,7 +679,7 @@ int main(int argc, char *argv[]) { TAOS_RES* res = taos_query(taos, command); taos_free_result(res); - sprintf(command, "create database %s;", db_name); + sprintf(command, "create database %s replica %d;", db_name, replica); res = taos_query(taos, command); taos_free_result(res); From 72a1a4bb46782c1de918cff4540377ef6d80d924 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 24 Aug 2020 16:55:58 +0800 Subject: [PATCH 23/44] [update version info] --- src/util/src/version.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util/src/version.c b/src/util/src/version.c index ac5de90dc8..6f7b29ebb9 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,7 +1,7 @@ -char version[12] = "2.0.1.1"; +char version[12] = "2.0.2.0"; char compatible_version[12] = "2.0.0.0"; -char gitinfo[48] = "ae1966332948147bacce3d32f9ad539ab8721db2"; -char gitinfoOfInternal[48] = "bf53767db56cedb1c484df83a1f10536f12647ad"; -char buildinfo[64] = "Built by root at 2020-08-20 15:46"; +char gitinfo[48] = "d711657139620f6c50f362597020705b8ad26bd2"; +char gitinfoOfInternal[48] = "1d74ae24c541ffbb280e8630883c0236cd45f8c7"; +char buildinfo[64] = "Built by root at 2020-08-24 16:31"; -void libtaos_2_0_1_1_Linux_x64() {}; +void libtaos_2_0_2_0_Linux_x64_beta() {}; From ee5038ea85fa352de3ae710afe511ec07c4b96ea Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 24 Aug 2020 18:12:43 +0800 Subject: [PATCH 24/44] [TD-880] --- src/os/inc/osDir.h | 2 +- src/os/src/detail/osDir.c | 15 +++++++-------- src/vnode/src/vnodeMain.c | 6 ++++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index 73e4b216e6..e7dc04fd15 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -23,7 +23,7 @@ extern "C" { // TAOS_OS_FUNC_DIR void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); -void taosMvDir(char* destDir, char *srcDir); +void taosRename(char* oldName, char *newName); #ifdef __cplusplus } diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 9496b74405..2f82d8866c 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -50,18 +50,17 @@ int taosMkDir(const char *path, mode_t mode) { return code; } -void taosMvDir(char* destDir, char *srcDir) { +void taosRename(char* oldName, char *newName) { if (0 == tsEnableVnodeBak) { uInfo("vnode backup not enabled"); return; } - char shellCmd[1024+1] = {0}; - - //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); - (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); - taosSystem(shellCmd); - uInfo("shell cmd:%s is executed", shellCmd); + if (rename(oldName, newName)) { + uError("%s is modify to %s fail, reason:%s", oldName, newName, strerror(errno)); + } else { + uInfo("%s is modify to %s success!", oldName, newName); + } } -#endif \ No newline at end of file +#endif diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 0a5e292f6d..254c10b31e 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -357,9 +357,11 @@ void vnodeRelease(void *pVnodeRaw) { taosTFree(pVnode->rootDir); if (pVnode->dropped) { - char rootDir[TSDB_FILENAME_LEN] = {0}; + char rootDir[TSDB_FILENAME_LEN] = {0}; + char newDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); - taosMvDir(tsVnodeBakDir, rootDir); + sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId); + taosRename(rootDir, newDir); taosRemoveDir(rootDir); } From c6c3d96cb50924e4c4cff0157079fbe5639c7781 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 24 Aug 2020 18:18:48 +0800 Subject: [PATCH 25/44] [TD-880] --- src/os/src/detail/osDir.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 2f82d8866c..7a537cdfea 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -56,6 +56,8 @@ void taosRename(char* oldName, char *newName) { return; } + // if newName in not empty, rename return fail. + // the newName must be empty or does not exist if (rename(oldName, newName)) { uError("%s is modify to %s fail, reason:%s", oldName, newName, strerror(errno)); } else { From 1b3e5d0f0c9c7c857a4666fcbbdc51ab05fb7f7f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 15:03:19 +0000 Subject: [PATCH 26/44] TD-1216 update normal table uid --- src/mnode/src/mnodeTable.c | 5 +- tests/test/c/CMakeLists.txt | 11 +- tests/test/c/createNormalTable.c | 218 +++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 tests/test/c/createNormalTable.c diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index dbc1bffa5c..8506501194 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1714,7 +1714,8 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->superTable = pMsg->pSTable; } else { - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t)pTable->vgId) << 40) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 16) + + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); pTable->sqlLen = htons(pCreate->sqlLen); @@ -1744,7 +1745,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; - mDebug("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mInfo("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->sqlLen, pTable->sql); } } diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index cc8bc78e33..9d11895104 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -10,8 +10,8 @@ IF (TD_LINUX) #add_executable(insertPerTable insertPerTable.c) #target_link_libraries(insertPerTable taos_static pthread) - add_executable(insertPerRow insertPerRow.c) - target_link_libraries(insertPerRow taos_static pthread) + #add_executable(insertPerRow insertPerRow.c) + #target_link_libraries(insertPerRow taos_static pthread) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) @@ -22,6 +22,9 @@ IF (TD_LINUX) #add_executable(hashPerformance hashPerformance.c) #target_link_libraries(hashPerformance taos_static tutil common pthread) - add_executable(createTablePerformance createTablePerformance.c) - target_link_libraries(createTablePerformance taos_static tutil common pthread) + #add_executable(createTablePerformance createTablePerformance.c) + #target_link_libraries(createTablePerformance taos_static tutil common pthread) + + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/createNormalTable.c b/tests/test/c/createNormalTable.c new file mode 100644 index 0000000000..18a648b9e1 --- /dev/null +++ b/tests/test/c/createNormalTable.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "tglobal.h" +#include "hash.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t numOfThreads = 30; +int32_t numOfTables = 100000; +int32_t replica = 1; +int32_t numOfColumns = 2; + +typedef struct { + int32_t tableBeginIndex; + int32_t tableEndIndex; + int32_t threadIndex; + char dbName[32]; + char stableName[64]; + float createTableSpeed; + pthread_t thread; +} SThreadInfo; + +void shellParseArgument(int argc, char *argv[]); +void *threadFunc(void *param); +void createDbAndSTable(); + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + taos_init(); + createDbAndSTable(); + + pPrint("%d threads are spawned to create table", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + int32_t numOfTablesPerThread = numOfTables / numOfThreads; + numOfTables = numOfTablesPerThread * numOfThreads; + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + pPrint("%s total speed:%.1f tables/second, threads:%d %s", GREEN, createTableSpeed, numOfThreads, NC); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void createDbAndSTable() { + pPrint("start to create db and stable"); + char qstr[64000]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s replica %d", dbName, replica); + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + sprintf(qstr, "use %s", dbName); + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + taos_close(con); +} + +void *threadFunc(void *param) { + SThreadInfo *pInfo = (SThreadInfo *)param; + char qstr[65000]; + int code; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + TAOS_RES *pSql = taos_query(con, qstr); + taos_free_result(pSql); + + int64_t startMs = taosGetTimestampMs(); + + for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + } + taos_free_result(pSql); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + int64_t endMs = taosGetTimestampMs(); + int32_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + float seconds = (endMs - startMs) / 1000.0; + float speed = totalTables / seconds; + pInfo->createTableSpeed = speed; + + pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed); + taos_close(con); + + return 0; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "replica, default is ", replica); + printf("%s%s\n", indent, "-columns"); + printf("%s%s%s%d\n", indent, indent, "numOfColumns, default is ", numOfColumns); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + replica = atoi(argv[++i]); + } else if (strcmp(argv[i], "-columns") == 0) { + numOfColumns = atoi(argv[++i]); + } else { + } + } + + pPrint("%s dbName:%s %s", GREEN, dbName, NC); + pPrint("%s stableName:%s %s", GREEN, stableName, NC); + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s numOfTables:%d %s", GREEN, numOfTables, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s numOfColumns:%d %s", GREEN, numOfColumns, NC); + pPrint("%s replica:%d %s", GREEN, replica, NC); + + pPrint("%s start create table performace test %s", GREEN, NC); +} From 99715c7b52274021d498db9ed1b7b41368364c0d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 15:06:47 +0000 Subject: [PATCH 27/44] TD-1217 let monitor database can be dropped --- src/mnode/src/mnodeDb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 26b4430777..0367ce21fd 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1059,10 +1059,12 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { } } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("db:%s, can't drop monitor database", pDrop->db); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif int32_t code = mnodeSetDbDropping(pMsg->pDb); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { From 3fb64ca0e21547b79751bf6cfe475d479a61a5d0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 15:11:29 +0000 Subject: [PATCH 28/44] minor changes --- src/mnode/src/mnodeTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 8506501194..794958f7f0 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1745,7 +1745,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; - mInfo("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mDebug("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->sqlLen, pTable->sql); } } From 69ebb401deb40cf2c94738f72943107ce2628be2 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Mon, 24 Aug 2020 23:18:57 +0800 Subject: [PATCH 29/44] Update faq-ch.md add new faq about JDBC --- documentation20/webdocs/markdowndocs/faq-ch.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index ba2d91c4ac..de47f376ac 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -68,5 +68,13 @@ Connection = DriverManager.getConnection(url, properties); ## 12.TDengine GO windows驱动的如何编译? 请看为此问题撰写的技术博客 - +## 13.JDBC报错: the excuted SQL is not a DML or a DDL? +请更新至最新的JDBC驱动 +```JAVA + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.4 + +``` From c43cca3cb4ee785a9814899479bb0c34ae2df5f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 25 Aug 2020 11:35:56 +0800 Subject: [PATCH 30/44] minor changes --- tests/script/general/user/monitor.sim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index 208b85f16c..eb543612f5 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -11,9 +11,9 @@ sleep 3000 sql connect print ========== step2 -sql drop database log -x step21 - return -1 -step21: +#sql drop database log -x step21 +# return -1 +#step21: sql drop table log.dn -x step22 return -1 step22: From a8417e7b6e162fd3ab84f3bf0740a6c046229e2c Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 13:48:32 +0800 Subject: [PATCH 31/44] [TD-1203] --- packaging/release.sh | 7 +- packaging/tools/install.sh | 74 ++++---- packaging/tools/install_arbi.sh | 297 ++++++++++++++++++++++++++++++++ packaging/tools/makearbi.sh | 75 ++++++++ packaging/tools/remove.sh | 12 +- packaging/tools/remove_arbi.sh | 129 ++++++++++++++ 6 files changed, 545 insertions(+), 49 deletions(-) mode change 100644 => 100755 packaging/tools/install.sh create mode 100755 packaging/tools/install_arbi.sh create mode 100755 packaging/tools/makearbi.sh create mode 100755 packaging/tools/remove_arbi.sh diff --git a/packaging/release.sh b/packaging/release.sh index 3ba1ba72b2..fe46c97928 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -80,9 +80,9 @@ fi versioninfo="${top_dir}/src/util/src/version.c" csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" -fi +#if command -v sudo > /dev/null; then +# csudo="sudo" +#fi function is_valid_version() { [ -z $1 ] && return 1 || : @@ -267,6 +267,7 @@ if [ "$osType" != "Darwin" ]; then ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else cd ${script_dir}/tools ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh old mode 100644 new mode 100755 index 64de09df6d..f38677f7ac --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -76,7 +76,7 @@ fi # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ -d /etc/os-release ]]; then +if [[ -e /etc/os-release ]]; then osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: else osinfo="" @@ -381,34 +381,29 @@ function install_service_on_sysvinit() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/taosd.service" - if systemctl is-active --quiet taosd; then echo "TDengine is running, stopping it..." ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${taosd_service_config} - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null - - ${csudo} rm -f ${tarbitratord_service_config} fi } @@ -418,7 +413,6 @@ function install_service_on_systemd() { clean_service_on_systemd taosd_service_config="${service_config_dir}/taosd.service" - ${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" ${csudo} bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" @@ -439,32 +433,30 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" ${csudo} systemctl enable taosd - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + if [ "$verMode" == "cluster" ]; then - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - - ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh new file mode 100755 index 0000000000..662d2dfae1 --- /dev/null +++ b/packaging/tools/install_arbi.sh @@ -0,0 +1,297 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/taos" + +# old bin dir +bin_dir="/usr/local/taos/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install taosd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +# taos:2345:respawn:/etc/init.d/tarbitratord start + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop taosd + kill_tarbitrator + fi +} + +function update_TDengine() { + # Start to update + echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" +} + +function install_TDengine() { + # Start to install + echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_TDengine +else + install_TDengine +fi + diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh new file mode 100755 index 0000000000..bc6179eff2 --- /dev/null +++ b/packaging/tools/makearbi.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TDengine-enterprise-arbitrator" +else + install_dir="${release_dir}/TDengine-arbitrator" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" +install_files="${script_dir}/install_arbi.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index c775f98b83..dd6fe65eb7 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -69,10 +69,12 @@ function kill_tarbitrator() { } function clean_bin() { # Remove link - ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosd || : - ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/taos || : + ${csudo} rm -f ${bin_link_dir}/taosd || : + ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : } function clean_lib() { @@ -207,7 +209,7 @@ ${csudo} rm -rf ${data_link_dir} || : ${csudo} rm -rf ${install_main_dir} ${csudo} rm -rf ${install_nginxd_dir} -if [[ -d /etc/os-release ]]; then +if [[ -e /etc/os-release ]]; then osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) else osinfo="" diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh new file mode 100755 index 0000000000..127da68fcd --- /dev/null +++ b/packaging/tools/remove_arbi.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/taos" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TDengine's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}" From 37da0e660f55312a4c5190216af4c60b616fd8b0 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:13:02 +0800 Subject: [PATCH 32/44] [TD-1203] --- src/inc/taosdef.h | 1 + src/sync/src/tarbitrator.c | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..ff5b9a958f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -395,6 +395,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 #define TAOS_QTYPE_RPC 0 #define TAOS_QTYPE_FWD 1 diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 3538391a94..625c0d6838 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -40,12 +40,16 @@ typedef struct { void *pConn; } SNodeConn; +uint16_t tsArbitratorPort = 0; + int main(int argc, char *argv[]) { char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; + tsArbitratorPort = tsServerPort + TSDB_PORT_ARBITRATOR; + for (int i=1; i Date: Tue, 25 Aug 2020 14:31:15 +0800 Subject: [PATCH 33/44] [TD-1203] --- packaging/tools/install_arbi.sh | 4 ++-- packaging/tools/remove_arbi.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index 662d2dfae1..a89d2257dc 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -13,10 +13,10 @@ bin_link_dir="/usr/bin" #inc_link_dir="/usr/include" #install main path -install_main_dir="/usr/local/taos" +install_main_dir="/usr/local/tarbitrator" # old bin dir -bin_dir="/usr/local/taos/bin" +bin_dir="/usr/local/tarbitrator/bin" service_config_dir="/etc/systemd/system" diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh index 127da68fcd..68fd9275fb 100755 --- a/packaging/tools/remove_arbi.sh +++ b/packaging/tools/remove_arbi.sh @@ -12,7 +12,7 @@ GREEN='\033[1;32m' NC='\033[0m' #install main path -install_main_dir="/usr/local/taos" +install_main_dir="/usr/local/tarbitrator" bin_link_dir="/usr/bin" #inc_link_dir="/usr/include" From b3b3108e5bebc7c5e1d295bf8270746df7d3bead Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:53:10 +0800 Subject: [PATCH 34/44] [TD-1203] --- packaging/cfg/taos.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 93a6936da3..a91dde8796 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -24,7 +24,7 @@ # dataDir /var/lib/taos # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only -# arbitrator arbitrator_hostname:6030 +# arbitrator arbitrator_hostname:6042 # number of threads per CPU core # numOfThreadsPerCore 1.0 From 24d33d02436f72de2ba2a13f083465a189ae2a0f Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:56:43 +0800 Subject: [PATCH 35/44] [TD-1203] --- documentation20/webdocs/markdowndocs/administrator-ch.md | 2 +- documentation20/webdocs/markdowndocs/cluster-ch.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 97f9f9e839..c5ecb862e3 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -94,7 +94,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修 - maxSQLLength:单条SQL语句允许最长限制。默认值:65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息,0表示不允许,1表示允许。 默认值:1。 -**注意:**对于端口,TDengine会使用从serverPort起12个连续的TCP和UDP端口号,请务必在防火墙打开。因此如果是缺省配置,需要打开从6030都6041共12个端口,而且必须TCP和UDP都打开。 +**注意:**对于端口,TDengine会使用从serverPort起13个连续的TCP和UDP端口号,请务必在防火墙打开。因此如果是缺省配置,需要打开从6030都6042共13个端口,而且必须TCP和UDP都打开。 不同应用场景的数据往往具有不同的数据特征,比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率,TDengine提供如下存储相关的系统配置参数: diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 31d70e73e1..097433a18a 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -10,7 +10,7 @@ TDengine的集群管理极其简单,除添加和删除节点需要人工干预 **第一步**:如果搭建集群的节点中,存有之前的测试数据、装过1.X的版本,或者装过其他版本的TDengine,请先将其删除,并清空所有数据,具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -**第二步**:建议关闭防火墙,至少保证端口:6030 - 6041的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口; +**第二步**:建议关闭防火墙,至少保证端口:6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口; **第三步**:在所有节点安装TDengine,且版本必须是一致的,**但不要启动taosd**; @@ -33,7 +33,7 @@ fqdn h1.taosdata.com serverPort 6030 // 副本数为偶数的时候,需要配置,请参考《Arbitrator的使用》的部分 -arbitrator ha.taosdata.com:6030 +arbitrator ha.taosdata.com:6042 ``` 一定要修改的参数是firstEp, 其他参数可不做任何修改,除非你很清楚为什么要修改。 @@ -169,5 +169,5 @@ SHOW MNODES; 如果副本数为偶数,当一个vnode group里一半或超过一半的vnode不工作时,是无法从中选出master的。同理,一半或超过一半的mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。 -TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6030。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator。 +TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6042。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator。 From 6cca3cdcb929e1c27385250bf5b1c65d086b411f Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 15:00:40 +0800 Subject: [PATCH 36/44] [TD-1121] --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index e9d7a71477..eb6fcde385 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -54,7 +54,7 @@ int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 1; +int32_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; /* From dffb6511e5c83194151fbef644d95877e5089a24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 16:04:00 +0800 Subject: [PATCH 37/44] [td-1227] --- src/mnode/src/mnodeProfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 01a824baa7..85457d7a26 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -35,8 +35,8 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" -#define CONN_KEEP_TIME (tsShellActivityTimer * 3000) -#define CONN_CHECK_TIME (tsShellActivityTimer * 2000) +#define CONN_KEEP_TIME (tsShellActivityTimer * 3) +#define CONN_CHECK_TIME (tsShellActivityTimer * 2) #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 From 308998cd223cfd87076c0b4351e476bfe9596e6d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 25 Aug 2020 16:50:45 +0800 Subject: [PATCH 38/44] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index de47f376ac..5e247a796d 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -77,4 +77,10 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` +## 14. 怎么报告问题? +如果 FAQ 中的信息不能够帮到您,需要 TDengine 技术团队的技术支持与协助,请将以下两个目录中内容打包: +1. /var/log/taos +2. /etc/taos +附上必要的问题描述,以及发生该问题的执行操作,出现问题的表征及大概的时间,在 GitHub提交Issue。 +为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。 From 6aefd4963031ab2587387f70a137a521ddf82408 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 25 Aug 2020 17:17:32 +0800 Subject: [PATCH 39/44] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 5e247a796d..928096dac3 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -83,4 +83,4 @@ Connection = DriverManager.getConnection(url, properties); 2. /etc/taos 附上必要的问题描述,以及发生该问题的执行操作,出现问题的表征及大概的时间,在 GitHub提交Issue。 -为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。 +为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。但系统正常运行时,请一定将debugFlag设置为131,否则会产生大量的日志信息,降低系统效率。 From a373658308be3af32574aeb1b71f54630fa6b6b8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 25 Aug 2020 21:35:15 +0800 Subject: [PATCH 40/44] change to v16/v20 instead of master/develop --- .../perftest-scripts/tdengineTestWriteLoop.sh | 86 +++++++++++++------ 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index e1ee4418dc..cb9d87d0c1 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -24,22 +24,46 @@ function runTest { for r in ${!rowsPerRequest[@]}; do for c in `seq 1 $clients`; do totalRPR=0 - OUTPUT_FILE=tdengineTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out + if $v16 ; then + OUTPUT_FILE=tdengineTestWrite-v16-RPR${rowsPerRequest[$r]}-clients$c.out + else + OUTPUT_FILE=tdengineTestWrite-v20-RPR${rowsPerRequest[$r]}-clients$c.out + fi for i in `seq 1 $NUM_LOOP`; do - restartTaosd - $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 - printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + if ! $printresultonly ; then + restartTaosd + $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 + + if $v16 ; then + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]}" - $TDTEST_DIR/tdengineTest \ - -dataDir $DATA_DIR \ - -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]} \ - | tee $OUTPUT_FILE + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + else + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + fi + fi + RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` printTo "rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR" @@ -86,25 +110,30 @@ function restartTaosd { ################ Main ################ -master=false -develop=true +v16=false +v20=true verbose=false clients=1 +printresultonly=false while : ; do case $1 in + printresultonly) + printresultonly=true + shift ;; + -v) verbose=true shift ;; - master) - master=true - develop=false + v16) + v16=true + v20=false shift ;; - develop) - master=false - develop=true + v20) + v16=false + v20=true shift ;; -c) @@ -120,19 +149,24 @@ while : ; do esac done -if $master ; then - echo "Test master branch.." - cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg - WORK_DIR=/mnt/root/TDengine.master +if $v16 ; then + echo "Test v16 branch.." + WORK_DIR=/mnt/root/TDengine.v16 + cp /mnt/root/cfg/v16/taos.cfg /etc/taos/taos.cfg else - echo "Test develop branch.." - cp /mnt/root/cfg/develop/taos.cfg /etc/taos/taos.cfg + echo "Test v20 branch.." + cp /mnt/root/cfg/v20/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine fi TAOSD_DIR=$WORK_DIR/debug/build/bin TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine +if [ ! -f $TDTEST_DIR/tdengineTest ]; then + echo "Please build tdengineTest first!" + exit 1 +fi + runTest echo "Test done!" From 60a1f46bfbbf8075b986939c8277cfaea034e381 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 25 Aug 2020 22:25:46 +0800 Subject: [PATCH 41/44] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 928096dac3..4a3b88a256 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -81,6 +81,7 @@ Connection = DriverManager.getConnection(url, properties); 如果 FAQ 中的信息不能够帮到您,需要 TDengine 技术团队的技术支持与协助,请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos + 附上必要的问题描述,以及发生该问题的执行操作,出现问题的表征及大概的时间,在 GitHub提交Issue。 为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。但系统正常运行时,请一定将debugFlag设置为131,否则会产生大量的日志信息,降低系统效率。 From 5518b07395a737ff55240b2b31e93c2b30530fd4 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 06:45:21 +0800 Subject: [PATCH 42/44] Update faq-ch.md --- .../webdocs/markdowndocs/faq-ch.md | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 928096dac3..2e2a2180a2 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -36,25 +36,34 @@ 检查客户端侧TCP端口链接是否工作:`nc {hostIP} {port}` -## 6. 虽然语法正确,为什么我还是得到 "Invalid SQL" 错误 +## 6. 遇到错误“Unexpected generic error in RPC”, 我怎么办? +产生这个错误,是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用,请做如下检查: + +1. 请检查连接的服务器的FQDN是否正确 +2. 如果网络配置有DNS server, 请检查是否正常工作 +3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件,查看该FQDN是否配置,并是否有正确的IP地址。 +4. 如果网络配置OK,从客户端所在机器,你需要能Ping该连接的FQDN,否则客户端是无法链接服务器的 + + +## 7. 虽然语法正确,为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确,2.0之前版本,请检查SQL语句长度是否超过64K。如果超过,也会返回这个错误。 -## 7. 是否支持validation queries? +## 8. 是否支持validation queries? TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。 -## 8. 我可以删除或更新一条记录吗? +## 9. 我可以删除或更新一条记录吗? 不能。因为TDengine是为联网设备采集的数据设计的,不容许修改。但TDengine提供数据保留策略,只要数据记录超过保留时长,就会被自动删除。 -## 10. 我怎么创建超过250列的表? +## 10. 我怎么创建超过1024列的表? 使用2.0及其以上版本,默认支持1024列;2.0之前的版本,TDengine最大允许创建250列的表。但是如果确实超过限值,建议按照数据特性,逻辑地将这个宽表分解成几个小表。 ## 10. 最有效的写入数据的方法是什么? -批量插入。每条写入语句可以一张表同时插入多条记录,也可以同时插入多张表的记录。 +批量插入。每条写入语句可以一张表同时插入多条记录,也可以同时插入多张表的多条记录。 ## 11. 最有效的写入数据的方法是什么?windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决? @@ -77,7 +86,7 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 14. 怎么报告问题? +## 15. 怎么报告问题? 如果 FAQ 中的信息不能够帮到您,需要 TDengine 技术团队的技术支持与协助,请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos From 9efe5fe06ad0077c18aae2ee13bbe25d390e646a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 00:00:58 +0000 Subject: [PATCH 43/44] minor changes : --- documentation20/webdocs/markdowndocs/faq-ch.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 2e2a2180a2..b760fe161a 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -86,10 +86,11 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 15. 怎么报告问题? +## 14. 怎么报告问题? 如果 FAQ 中的信息不能够帮到您,需要 TDengine 技术团队的技术支持与协助,请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos + 附上必要的问题描述,以及发生该问题的执行操作,出现问题的表征及大概的时间,在 GitHub提交Issue。 为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。但系统正常运行时,请一定将debugFlag设置为131,否则会产生大量的日志信息,降低系统效率。 From 5e17894fbae8932ba8a70c0caad1d2ff0f5847e8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 14:33:03 +0800 Subject: [PATCH 44/44] Update architecture-ch.md --- documentation20/webdocs/markdowndocs/architecture-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index bfe3b55bd2..7ab4b5d096 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -82,7 +82,7 @@ TDengine 分布式架构的逻辑结构图如下: ### 节点之间的通讯 **通讯方式:**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景,数据写入的包一般不大,因此TDengine 除采用TCP做传输之外,还采用UDP方式,因为UDP 更加高效,而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制,以确保UDP的可靠传输。对于数据量不到15K的数据包,采取UDP的方式进行传输,超过15K的,或者是查询类的操作,自动采取TCP的方式进行传输。同时,TDengine根据配置和数据包,会自动对数据进行压缩/解压缩,数字签名/认证等处理。对于数据节点之间的数据复制,只采用TCP方式进行数据传输。 -**FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过选项“fqdn"进行指定,如果没有指定,系统将自动获取FQDN。如果节点没有配置FQDN,可以直接使用IP地址作为FQDN,但不建议使用,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。 +**FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过参数“fqdn"进行指定,如果没有指定,系统将自动获取FQDN。如果节点没有配置FQDN,可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN,需要保证DNS服务正常工作,或者在节点以及应用所在的节点配置好hosts文件。 **端口配置:**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定,对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口,是serverPort+10. 为支持多线程高效的处理UDP数据,每个对内和对外的UDP链接,都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10,总共11个TCP/UDP端口。使用时,需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。