From ababaa75e390094a38f70c15c49753a31c063dc8 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 10:23:33 +0800 Subject: [PATCH 01/45] [TD-1702]: add test cases for TSDBDriver --- .../java/com/taosdata/jdbc/TSDBDriver.java | 208 +++++++---------- .../com/taosdata/jdbc/TSDBDriverTest.java | 212 +++++++++++++++--- 2 files changed, 267 insertions(+), 153 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 97d93fb0a1..cd45048a77 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -14,13 +14,9 @@ *****************************************************************************/ package com.taosdata.jdbc; - import java.io.*; - import java.sql.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; +import java.util.*; import java.util.logging.Logger; /** @@ -44,76 +40,53 @@ import java.util.logging.Logger; */ public class TSDBDriver implements java.sql.Driver { - @Deprecated private static final String URL_PREFIX1 = "jdbc:TSDB://"; private static final String URL_PREFIX = "jdbc:TAOS://"; - /** - * Key used to retrieve the database value from the properties instance passed - * to the driver. - */ - public static final String PROPERTY_KEY_DBNAME = "dbname"; - /** * Key used to retrieve the host value from the properties instance passed to * the driver. */ public static final String PROPERTY_KEY_HOST = "host"; - /** - * Key used to retrieve the password value from the properties instance passed - * to the driver. - */ - public static final String PROPERTY_KEY_PASSWORD = "password"; - /** * Key used to retrieve the port number value from the properties instance * passed to the driver. */ public static final String PROPERTY_KEY_PORT = "port"; - + /** + * Key used to retrieve the database value from the properties instance passed + * to the driver. + */ + public static final String PROPERTY_KEY_DBNAME = "dbname"; /** * Key used to retrieve the user value from the properties instance passed to * the driver. */ public static final String PROPERTY_KEY_USER = "user"; - - + /** + * Key used to retrieve the password value from the properties instance passed + * to the driver. + */ + public static final String PROPERTY_KEY_PASSWORD = "password"; /** * Key for the configuration file directory of TSDB client in properties instance */ public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; - /** * Key for the timezone used by the TSDB client in properties instance */ public static final String PROPERTY_KEY_TIME_ZONE = "timezone"; - /** * Key for the locale used by the TSDB client in properties instance */ public static final String PROPERTY_KEY_LOCALE = "locale"; - - /** * Key for the char encoding used by the TSDB client in properties instance */ public static final String PROPERTY_KEY_CHARSET = "charset"; - public static final String PROPERTY_KEY_PROTOCOL = "protocol"; - - - /** - * Index for port coming out of parseHostPortPair(). - */ - public final static int PORT_NUMBER_INDEX = 1; - - /** - * Index for host coming out of parseHostPortPair(). - */ - public final static int HOST_NAME_INDEX = 0; - private TSDBDatabaseMetaData dbMetaData = null; static { @@ -169,9 +142,11 @@ public class TSDBDriver implements java.sql.Driver { } public Connection connect(String url, Properties info) throws SQLException { - if (url == null) { + if (url == null) throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!")); - } + + if (!acceptsURL(url)) + return null; Properties props = null; if ((props = parseURL(url, info)) == null) { @@ -179,7 +154,10 @@ public class TSDBDriver implements java.sql.Driver { } //load taos.cfg start - if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) { + if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null || + info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && ( + info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null || + info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) { File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR)); File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0]; List endpoints = loadConfigEndpoints(cfgFile); @@ -190,7 +168,9 @@ public class TSDBDriver implements java.sql.Driver { } try { - TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET), + TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), + (String) props.get(PROPERTY_KEY_LOCALE), + (String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE)); Connection newConn = new TSDBConnection(props, this.dbMetaData); return newConn; @@ -208,43 +188,15 @@ public class TSDBDriver implements java.sql.Driver { } /** - * Parses hostPortPair in the form of [host][:port] into an array, with the - * element of index HOST_NAME_INDEX being the host (or null if not specified), - * and the element of index PORT_NUMBER_INDEX being the port (or null if not - * specified). - * - * @param hostPortPair host and port in form of of [host][:port] - * @return array containing host and port as Strings - * @throws SQLException if a parse error occurs + * @param url the URL of the database + * @return true if this driver understands the given URL; + * false otherwise + * @throws SQLException if a database access error occurs or the url is {@code null} */ - protected static String[] parseHostPortPair(String hostPortPair) throws SQLException { - String[] splitValues = new String[2]; - - int portIndex = hostPortPair.indexOf(":"); - - String hostname = null; - - if (portIndex != -1) { - if ((portIndex + 1) < hostPortPair.length()) { - String portAsString = hostPortPair.substring(portIndex + 1); - hostname = hostPortPair.substring(0, portIndex); - - splitValues[HOST_NAME_INDEX] = hostname; - - splitValues[PORT_NUMBER_INDEX] = portAsString; - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("port is not proper!")); - } - } else { - splitValues[HOST_NAME_INDEX] = hostPortPair; - splitValues[PORT_NUMBER_INDEX] = null; - } - - return splitValues; - } - public boolean acceptsURL(String url) throws SQLException { - return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX); + if (url == null) + throw new SQLException(TSDBConstants.WrapErrMsg("url is null")); + return (url != null && url.length() > 0 && url.trim().length() > 0) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1)); } public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { @@ -252,15 +204,17 @@ public class TSDBDriver implements java.sql.Driver { info = new Properties(); } - if ((url != null) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1))) { + if (acceptsURL(url)) { info = parseURL(url, info); } DriverPropertyInfo hostProp = new DriverPropertyInfo(PROPERTY_KEY_HOST, info.getProperty(PROPERTY_KEY_HOST)); - hostProp.required = true; + hostProp.required = false; + hostProp.description = "Hostname"; DriverPropertyInfo portProp = new DriverPropertyInfo(PROPERTY_KEY_PORT, info.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT)); portProp.required = false; + portProp.description = "Port"; DriverPropertyInfo dbProp = new DriverPropertyInfo(PROPERTY_KEY_DBNAME, info.getProperty(PROPERTY_KEY_DBNAME)); dbProp.required = false; @@ -268,9 +222,11 @@ public class TSDBDriver implements java.sql.Driver { DriverPropertyInfo userProp = new DriverPropertyInfo(PROPERTY_KEY_USER, info.getProperty(PROPERTY_KEY_USER)); userProp.required = true; + userProp.description = "User"; DriverPropertyInfo passwordProp = new DriverPropertyInfo(PROPERTY_KEY_PASSWORD, info.getProperty(PROPERTY_KEY_PASSWORD)); passwordProp.required = true; + passwordProp.description = "Password"; DriverPropertyInfo[] propertyInfo = new DriverPropertyInfo[5]; propertyInfo[0] = hostProp; @@ -283,20 +239,60 @@ public class TSDBDriver implements java.sql.Driver { } /** - * example: jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password + * example: jdbc:TAOS://127.0.0.1:0/db?user=root&password=your_password */ - public Properties parseURL(String url, Properties defaults) throws java.sql.SQLException { + public Properties parseURL(String url, Properties defaults) { Properties urlProps = (defaults != null) ? defaults : new Properties(); - if (url == null) { + if (url == null || url.length() <= 0 || url.trim().length() <= 0) return null; - } - - if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) { + if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) return null; - } + // parse properties + int beginningOfSlashes = url.indexOf("//"); + int index = url.indexOf("?"); + if (index != -1) { + String paramString = url.substring(index + 1, url.length()); + url = url.substring(0, index); + StringTokenizer queryParams = new StringTokenizer(paramString, "&"); + while (queryParams.hasMoreElements()) { + String parameterValuePair = queryParams.nextToken(); + int indexOfEqual = parameterValuePair.indexOf("="); + String parameter = null; + String value = null; + if (indexOfEqual != -1) { + parameter = parameterValuePair.substring(0, indexOfEqual); + if (indexOfEqual + 1 < parameterValuePair.length()) { + value = parameterValuePair.substring(indexOfEqual + 1); + } + } + if ((value != null && value.length() > 0) && (parameter != null && parameter.length() > 0)) { + urlProps.setProperty(parameter, value); + } + } + } + // parse dbname + url = url.substring(beginningOfSlashes + 2); + int indexOfSlash = url.indexOf("/"); + if (indexOfSlash != -1) { + if (indexOfSlash + 1 < url.length()) { + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_DBNAME, url.substring(indexOfSlash + 1)); + } + url = url.substring(0, indexOfSlash); + } + // parse port + int indexOfColon = url.indexOf(":"); + if (indexOfColon != -1) { + if (indexOfColon + 1 < url.length()) { + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_PORT, url.substring(indexOfColon + 1)); + } + url = url.substring(0, indexOfColon); + } + if (url != null && url.length() > 0 && url.trim().length() > 0) { + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); + } + /* String urlForMeta = url; - String dbProductName = url.substring(url.indexOf(":") + 1); dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); int beginningOfSlashes = url.indexOf("//"); @@ -345,11 +341,11 @@ public class TSDBDriver implements java.sql.Driver { user = urlProps.getProperty(PROPERTY_KEY_USER).toString(); this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user); - +*/ return urlProps; } - public void setPropertyValue(Properties property, String[] keyValuePair) { + private void setPropertyValue(Properties property, String[] keyValuePair) { switch (keyValuePair[0].toLowerCase()) { case PROPERTY_KEY_USER: property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]); @@ -372,13 +368,12 @@ public class TSDBDriver implements java.sql.Driver { } } - public int getMajorVersion() { - return 1; + return 2; } public int getMinorVersion() { - return 1; + return 0; } public boolean jdbcCompliant() { @@ -389,33 +384,4 @@ public class TSDBDriver implements java.sql.Driver { return null; } - /** - * Returns the host property - * - * @param props the java.util.Properties instance to retrieve the hostname from. - * @return the host - */ - public String host(Properties props) { - return props.getProperty(PROPERTY_KEY_HOST, "localhost"); - } - - /** - * Returns the port number property - * - * @param props the properties to get the port number from - * @return the port number - */ - public int port(Properties props) { - return Integer.parseInt(props.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT)); - } - - /** - * Returns the database property from props - * - * @param props the Properties to look for the database property. - * @return the database name. - */ - public String database(Properties props) { - return props.getProperty(PROPERTY_KEY_DBNAME); - } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index a0981063a5..dc52fdadb7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -1,47 +1,195 @@ package com.taosdata.jdbc; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; -import java.sql.SQLException; +import java.sql.*; import java.util.Properties; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class TSDBDriverTest { + private static String[] validURLs = { + "jdbc:TAOS://localhost:0", + "jdbc:TAOS://localhost", + "jdbc:TAOS://localhost:6030/test", + "jdbc:TAOS://localhost:6030", + "jdbc:TAOS://localhost:6030/", + "jdbc:TSDB://localhost:6030", + "jdbc:TSDB://localhost:6030/", + "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata", + "jdbc:TAOS://:", + "jdbc:TAOS://:/", + "jdbc:TAOS://:/test", + "jdbc:TAOS://localhost:0/?user=root&password=taosdata" + }; + private static boolean islibLoaded; + + @BeforeClass + public static void before() { + String osName = System.getProperty("os.name").toLowerCase(); + String libPath = null; + switch (osName) { + case "linux": + libPath = "/usr/lib/libtaos.so"; + break; + case "windows": + libPath = "C:\\TDengine\\driver\\taos.dll"; + break; + default: + } + if (libPath == null) + return; + try { + System.loadLibrary(libPath); + } catch (UnsatisfiedLinkError error) { + islibLoaded = false; + } + islibLoaded = true; + } + @Test - public void urlParserTest() throws SQLException { + public void testParseURL() { TSDBDriver driver = new TSDBDriver(); - String url = "jdbc:TSDB://127.0.0.1:0/db"; - Properties properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), "db"); - assertEquals(properties.get("user"), "root"); - assertEquals(properties.get("password"), "your_password"); + String url = "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata&charset=UTF-8"; + Properties config = new Properties(); + Properties actual = driver.parseURL(url, config); + assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.get("host")); + assertEquals("failure - port should be 0", "0", actual.get("port")); + assertEquals("failure - dbname should be db", "db", actual.get("dbname")); + assertEquals("failure - user should be root", "root", actual.get("user")); + assertEquals("failure - password should be taosdata", "taosdata", actual.get("password")); + assertEquals("failure - charset should be UTF-8", "UTF-8", actual.get("charset")); - url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8"; - properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), "log"); - assertEquals(properties.get("charset"), "UTF-8"); - - url = "jdbc:TSDB://127.0.0.1:0/"; - properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), null); - - url = "jdbc:TSDB://127.0.0.1:0/db"; - properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), "db"); + url = "jdbc:TAOS://127.0.0.1:0"; + config = new Properties(); + actual = driver.parseURL(url, config); + assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host")); + assertEquals("failure - port should be 0", "0", actual.get("port")); + assertEquals("failure - dbname should be null", null, actual.get("dbname")); + + url = "jdbc:TAOS://127.0.0.1:0/db"; + config = new Properties(); + actual = driver.parseURL(url, config); + assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host")); + assertEquals("failure - port should be 0", "0", actual.get("port")); + assertEquals("failure - dbname should be db", "db", actual.get("dbname")); + + url = "jdbc:TAOS://:/?"; + config = new Properties(); + config.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + config.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); + actual = driver.parseURL(url, config); + assertEquals("failure - user should be root", "root", actual.getProperty("user")); + assertEquals("failure - password should be taosdata", "taosdata", actual.getProperty("password")); + assertEquals("failure - host should be null", null, actual.getProperty("host")); + assertEquals("failure - port should be null", null, actual.getProperty("port")); + assertEquals("failure - dbname should be null", null, actual.getProperty("dbname")); + } + + @Test + public void testConnectWithJdbcURL() { + + final String url = "jdbc:TAOS://localhost:3306/log?user=root&password=taosdata"; + try { + if (islibLoaded) { + Connection conn = DriverManager.getConnection(url); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testConnectWithProperties() { + final String jdbcUrl = "jdbc:TAOS://localhost.com:6030/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + try { + if (islibLoaded) { + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testConnectWithConfigFile() { + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + try { + if (islibLoaded) { + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLException.class) + public void testAcceptsURL() throws SQLException { + Driver driver = new TSDBDriver(); + for (String url : validURLs) { + assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); + } + new TSDBDriver().acceptsURL(null); + fail("acceptsURL throws exception when parameter is null"); + } + + @Test + public void testGetPropertyInfo() throws SQLException { + Driver driver = new TSDBDriver(); + final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, connProps); + for (DriverPropertyInfo info : propertyInfo) { + if (info.name.equals(TSDBDriver.PROPERTY_KEY_HOST)) + assertEquals("failure - host should be localhost", "localhost", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_PORT)) + assertEquals("failure - port should be 6030", "6030", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_DBNAME)) + assertEquals("failure - dbname should be test", "log", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_USER)) + assertEquals("failure - user should be root", "root", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_PASSWORD)) + assertEquals("failure - password should be root", "taosdata", info.value); + } + } + + @Test + public void testGetMajorVersion() throws SQLException { + Driver driver = new TSDBDriver(); + assertEquals("failure - getMajorVersion should be 2", 2, driver.getMajorVersion()); + } + + @Test + public void testGetMinorVersion() { + Driver driver = new TSDBDriver(); + assertEquals("failure - getMinorVersion should be 0", 0, driver.getMinorVersion()); + } + + @Test + public void testJdbcCompliant() { +// assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant()); + } + + @Test + public void testGetParentLogger() throws SQLFeatureNotSupportedException { + assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); } } \ No newline at end of file From 8a1c2b240a53144c61bc6f277b4d65946fc1bb34 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 10:34:15 +0800 Subject: [PATCH 02/45] change --- .../jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index dc52fdadb7..d5264e57d7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -44,10 +44,10 @@ public class TSDBDriverTest { return; try { System.loadLibrary(libPath); + islibLoaded = true; } catch (UnsatisfiedLinkError error) { islibLoaded = false; } - islibLoaded = true; } @Test From e7bb764bd303855cc84e8b694205325a62a382e3 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 14:23:42 +0800 Subject: [PATCH 03/45] change again --- .../com/taosdata/jdbc/TSDBDriverTest.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index d5264e57d7..677ed8b7a2 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -30,22 +29,27 @@ public class TSDBDriverTest { @BeforeClass public static void before() { String osName = System.getProperty("os.name").toLowerCase(); - String libPath = null; - switch (osName) { - case "linux": - libPath = "/usr/lib/libtaos.so"; - break; - case "windows": - libPath = "C:\\TDengine\\driver\\taos.dll"; - break; - default: - } - if (libPath == null) + if (!osName.equals("linux") && !osName.equals("windows")) { + islibLoaded = false; return; + } +// String libPath = null; +// switch (osName) { +// case "linux": +// libPath = "/usr/lib/libtaos.so"; +// break; +// case "windows": +// libPath = "C:\\TDengine\\driver\\taos.dll"; +// break; +// default: +// } +// if (libPath == null) +// return; try { - System.loadLibrary(libPath); + System.loadLibrary("taos"); islibLoaded = true; } catch (UnsatisfiedLinkError error) { + System.out.println("load tdengine lib failed."); islibLoaded = false; } } From ae26d9ac43b32cc390a3571be22b9458c9617c94 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 14:48:26 +0800 Subject: [PATCH 04/45] change --- .../com/taosdata/jdbc/TSDBDriverTest.java | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 677ed8b7a2..f7a200b33d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -3,6 +3,9 @@ package com.taosdata.jdbc; import org.junit.BeforeClass; import org.junit.Test; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.sql.*; import java.util.Properties; @@ -25,6 +28,7 @@ public class TSDBDriverTest { "jdbc:TAOS://localhost:0/?user=root&password=taosdata" }; private static boolean islibLoaded; + private static boolean isTaosdActived; @BeforeClass public static void before() { @@ -52,6 +56,21 @@ public class TSDBDriverTest { System.out.println("load tdengine lib failed."); islibLoaded = false; } + + try { + Process exec = Runtime.getRuntime().exec("ps -ef | grep taosd | grep -v \"grep\""); + BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); + int lineCnt = 0; + while (reader.readLine() != null) { + lineCnt++; + } + if (lineCnt > 0) + isTaosdActived = true; + else + isTaosdActived = false; + } catch (IOException e) { + isTaosdActived = false; + } } @Test @@ -96,7 +115,6 @@ public class TSDBDriverTest { @Test public void testConnectWithJdbcURL() { - final String url = "jdbc:TAOS://localhost:3306/log?user=root&password=taosdata"; try { if (islibLoaded) { @@ -104,7 +122,10 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { - e.printStackTrace(); + if (!isTaosdActived) + assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); + else + fail("failure - should not throw Exception"); } } @@ -121,7 +142,10 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { - e.printStackTrace(); + if (!isTaosdActived) + assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); + else + fail("failure - should not throw Exception"); } } @@ -138,7 +162,10 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { - e.printStackTrace(); + if (!isTaosdActived) + assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); + else + fail("failure - should not throw Exception"); } } From 97945d8c688dfb9e3ad788cf0a39ca598479913d Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 14:51:29 +0800 Subject: [PATCH 05/45] change --- .../jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index f7a200b33d..1b8f2f04d8 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -159,6 +159,7 @@ public class TSDBDriverTest { try { if (islibLoaded) { Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + System.out.println(conn); assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { From c6f6cd12bea98cf4cf921ef9447eacd272042e36 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 18:57:10 +0800 Subject: [PATCH 06/45] change --- .../com/taosdata/jdbc/TSDBDriverTest.java | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 1b8f2f04d8..8739c88e79 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -37,18 +37,6 @@ public class TSDBDriverTest { islibLoaded = false; return; } -// String libPath = null; -// switch (osName) { -// case "linux": -// libPath = "/usr/lib/libtaos.so"; -// break; -// case "windows": -// libPath = "C:\\TDengine\\driver\\taos.dll"; -// break; -// default: -// } -// if (libPath == null) -// return; try { System.loadLibrary("taos"); islibLoaded = true; @@ -58,16 +46,21 @@ public class TSDBDriverTest { } try { - Process exec = Runtime.getRuntime().exec("ps -ef | grep taosd | grep -v \"grep\""); - BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); - int lineCnt = 0; - while (reader.readLine() != null) { - lineCnt++; - } - if (lineCnt > 0) - isTaosdActived = true; - else + if (osName.equals("linux")) { + String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; + Process exec = Runtime.getRuntime().exec(cmd); + BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); + int lineCnt = 0; + while (reader.readLine() != null) { + lineCnt++; + } + if (lineCnt > 0) + isTaosdActived = true; + else + isTaosdActived = false; + } else { isTaosdActived = false; + } } catch (IOException e) { isTaosdActived = false; } @@ -146,6 +139,7 @@ public class TSDBDriverTest { assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); else fail("failure - should not throw Exception"); + e.printStackTrace(); } } From 8b2e3c82510b8ae3e98c7d45836493bd59ab1e28 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 19:01:05 +0800 Subject: [PATCH 07/45] change --- .../jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 8739c88e79..69fdeba110 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -108,7 +108,7 @@ public class TSDBDriverTest { @Test public void testConnectWithJdbcURL() { - final String url = "jdbc:TAOS://localhost:3306/log?user=root&password=taosdata"; + final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; try { if (islibLoaded) { Connection conn = DriverManager.getConnection(url); @@ -124,7 +124,7 @@ public class TSDBDriverTest { @Test public void testConnectWithProperties() { - final String jdbcUrl = "jdbc:TAOS://localhost.com:6030/test?user=root&password=taosdata"; + final String jdbcUrl = "jdbc:TAOS://localhost:6030/test?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); From e21088d57d23dc13017f69fd1416018b456cf3ea Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 19:09:50 +0800 Subject: [PATCH 08/45] change --- .../test/java/com/taosdata/jdbc/TSDBDriverTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 69fdeba110..e34531bddf 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -115,6 +115,7 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { + e.printStackTrace(); if (!isTaosdActived) assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); else @@ -135,11 +136,13 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { - if (!isTaosdActived) - assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); - else - fail("failure - should not throw Exception"); e.printStackTrace(); + if (!isTaosdActived) { + assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); + } + else { + fail("failure - should not throw Exception"); + } } } @@ -157,6 +160,7 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { + e.printStackTrace(); if (!isTaosdActived) assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); else From 60d04f8121b0ac92b4040fca4bb12223e6f2fbc5 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 19:12:57 +0800 Subject: [PATCH 09/45] change --- .../src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index e34531bddf..db05c57b58 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -125,7 +125,7 @@ public class TSDBDriverTest { @Test public void testConnectWithProperties() { - final String jdbcUrl = "jdbc:TAOS://localhost:6030/test?user=root&password=taosdata"; + final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -138,9 +138,9 @@ public class TSDBDriverTest { } catch (SQLException e) { e.printStackTrace(); if (!isTaosdActived) { + System.out.println(e.getMessage()); assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); - } - else { + } else { fail("failure - should not throw Exception"); } } @@ -148,7 +148,7 @@ public class TSDBDriverTest { @Test public void testConnectWithConfigFile() { - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); From dd2732ea68658996a7f0815ae79e0fc068d0a2c9 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 4 Nov 2020 19:15:27 +0800 Subject: [PATCH 10/45] change --- .../jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index db05c57b58..8718c3e7da 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -136,10 +136,8 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { - e.printStackTrace(); if (!isTaosdActived) { - System.out.println(e.getMessage()); - assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); + assertEquals("failure - should throw SQLException", SQLException.class, e.getClass()); } else { fail("failure - should not throw Exception"); } From 97f7cca4b6ac2ab73e59a5c09b4f86dc5f5cd286 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 19:39:03 +0800 Subject: [PATCH 11/45] [TD-225] refactor --- src/client/src/tscServer.c | 2 +- src/client/src/tscSubquery.c | 12 +- src/query/inc/qTsbuf.h | 36 +++--- src/query/src/qExecutor.c | 6 +- src/query/src/qTsbuf.c | 234 +++++++++++++++++------------------ 5 files changed, 145 insertions(+), 145 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1584afe706..b7d3a22596 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -882,7 +882,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pQueryInfo->tsBuf != NULL) { // note: here used the index instead of actual vnode id. int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; - int32_t code = dumpFileBlockByVnodeId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); + int32_t code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index f328a6c075..bb48ca89da 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -156,8 +156,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.id, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.id, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { pLimit->offset -= 1;//offset apply to projection? } @@ -193,8 +193,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ TSKEY et = taosGetTimestampUs(); tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", - pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfVnodes, win->skey, win->ekey, - tsBufGetNumOfVnodes(output1), et - st); + pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfGroups, win->skey, win->ekey, + tsBufGetNumOfGroup(output1), et - st); return output1->numOfTotal; } @@ -282,7 +282,7 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) { static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { int32_t num = 0; int32_t* list = NULL; - tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); + tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); // The virtual node, of which all tables are disqualified after the timestamp intersection, // is removed to avoid next stage query. @@ -314,7 +314,7 @@ static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { int32_t num = 0; int32_t* list = NULL; - tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); + tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); int32_t numOfGroups = taosArrayGetSize(pVgroupTables); diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 73b18f8915..90bd64336f 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -26,7 +26,7 @@ extern "C" { #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C -#define TS_COMP_FILE_VNODE_MAX 512 +#define TS_COMP_FILE_GROUP_MAX 512 typedef struct STSList { char* rawBuf; @@ -38,7 +38,7 @@ typedef struct STSList { typedef struct STSElem { TSKEY ts; tVariant* tag; - int32_t vnode; + int32_t id; } STSElem; typedef struct STSCursor { @@ -60,17 +60,17 @@ typedef struct STSBlock { * The size of buffer file should not be greater than 2G, * and the offset of int32_t type is enough */ -typedef struct STSVnodeBlockInfo { - int32_t vnode; // vnode id +typedef struct STSGroupBlockInfo { + int32_t id; // group id int32_t offset; // offset set value in file int32_t numOfBlocks; // number of total blocks int32_t compLen; // compressed size -} STSVnodeBlockInfo; +} STSGroupBlockInfo; -typedef struct STSVnodeBlockInfoEx { - STSVnodeBlockInfo info; +typedef struct STSGroupBlockInfoEx { + STSGroupBlockInfo info; int32_t len; // length before compress -} STSVnodeBlockInfoEx; +} STSGroupBlockInfoEx; typedef struct STSBuf { FILE* f; @@ -78,9 +78,9 @@ typedef struct STSBuf { uint32_t fileSize; // todo use array - STSVnodeBlockInfoEx* pData; + STSGroupBlockInfoEx* pData; uint32_t numOfAlloc; - uint32_t numOfVnodes; + uint32_t numOfGroups; char* assistBuf; int32_t bufSize; @@ -94,22 +94,22 @@ typedef struct STSBuf { typedef struct STSBufFileHeader { uint32_t magic; // file magic number - uint32_t numOfVnode; // number of vnode stored in current file + uint32_t numOfGroup; // number of group stored in current file int32_t tsOrder; // timestamp order in current file } STSBufFileHeader; STSBuf* tsBufCreate(bool autoDelete, int32_t order); STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t vnodeId); +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id); void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); +void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf); STSBuf* tsBufClone(STSBuf* pTSBuf); -STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId); +STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); void tsBufFlush(STSBuf* pTSBuf); @@ -118,7 +118,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf); bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); @@ -131,11 +131,11 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); */ void tsBufDisplay(STSBuf* pTSBuf); -int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf); +int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf); -void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId); +void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id); -int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks); +int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks); STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d65d01d155..26c4958ca5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3843,7 +3843,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); // failed to find data with the specified tag value and vnodeId - if (elem.vnode < 0) { + if (tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { @@ -4777,7 +4777,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->cur.vgroupIndex == -1) { STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId - if (elem.vnode < 0) { + if (tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { @@ -4802,7 +4802,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId - if (elem1.vnode < 0) { + if (tsBufIsValidElem(&elem1)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index df9ea1db03..dd5297bc78 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -4,7 +4,7 @@ #include "tutil.h" static int32_t getDataStartOffset(); -static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); @@ -32,7 +32,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSDB_ORDER_ASC}; + STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; STSBufUpdateHeader(pTSBuf, &header); tsBufResetPos(pTSBuf); @@ -75,9 +75,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if (header.numOfVnode > pTSBuf->numOfAlloc) { - pTSBuf->numOfAlloc = header.numOfVnode; - STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); + if (header.numOfGroup > pTSBuf->numOfAlloc) { + pTSBuf->numOfAlloc = header.numOfGroup; + STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -86,7 +86,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { pTSBuf->pData = tmp; } - pTSBuf->numOfVnodes = header.numOfVnode; + pTSBuf->numOfGroups = header.numOfGroup; // check the ts order pTSBuf->tsOrder = header.tsOrder; @@ -96,9 +96,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes; + size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize); + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -109,9 +109,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { UNUSED(sz); // the length value for each vnode is not kept in file, so does not set the length value - for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) { - STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i]; - memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo)); + for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { + STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; + memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } free(buf); @@ -131,8 +131,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->autoDelete = autoDelete; -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfVnode:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), -// pTSBuf->fileSize, pTSBuf->numOfVnodes, pTSBuf->autoDelete); +// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), +// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); return pTSBuf; } @@ -162,53 +162,53 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } -static STSVnodeBlockInfoEx* tsBufGetLastVnodeInfo(STSBuf* pTSBuf) { - int32_t last = pTSBuf->numOfVnodes - 1; +static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { + int32_t last = pTSBuf->numOfGroups - 1; assert(last >= 0); return &pTSBuf->pData[last]; } -static STSVnodeBlockInfoEx* addOneVnodeInfo(STSBuf* pTSBuf, int32_t vnodeId) { - if (pTSBuf->numOfAlloc <= pTSBuf->numOfVnodes) { +static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { + if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - STSVnodeBlockInfoEx* tmp = (STSVnodeBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } pTSBuf->pData = tmp; pTSBuf->numOfAlloc = newSize; - memset(&pTSBuf->pData[pTSBuf->numOfVnodes], 0, sizeof(STSVnodeBlockInfoEx) * (newSize - pTSBuf->numOfVnodes)); + memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); } - if (pTSBuf->numOfVnodes > 0) { - STSVnodeBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf); + if (pTSBuf->numOfGroups > 0) { + STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); // update prev vnode length info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pPrevBlockInfoEx->info); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); } // set initial value for vnode block - STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes].info; - pBlockInfo->vnode = vnodeId; + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; + pBlockInfo->id = id; pBlockInfo->offset = pTSBuf->fileSize; assert(pBlockInfo->offset >= getDataStartOffset()); // update vnode info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes, pBlockInfo); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); // add one vnode info - pTSBuf->numOfVnodes += 1; + pTSBuf->numOfGroups += 1; // update the header info STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; STSBufUpdateHeader(pTSBuf, &header); - return tsBufGetLastVnodeInfo(pTSBuf); + return tsBufGetLastGroupInfo(pTSBuf); } static void shrinkBuffer(STSList* ptsData) { @@ -279,10 +279,10 @@ static void writeDataToDisk(STSBuf* pTSBuf) { pTSBuf->tsData.len = 0; - STSVnodeBlockInfoEx* pVnodeBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf); + STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - pVnodeBlockInfoEx->info.compLen += blockSize; - pVnodeBlockInfoEx->info.numOfBlocks += 1; + pGroupBlockInfoEx->info.compLen += blockSize; + pGroupBlockInfoEx->info.numOfBlocks += 1; shrinkBuffer(&pTSBuf->tsData); } @@ -413,20 +413,20 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { - STSVnodeBlockInfoEx* pBlockInfo = NULL; +void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len) { + STSGroupBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; - if (pTSBuf->numOfVnodes == 0 || tsBufGetLastVnodeInfo(pTSBuf)->info.vnode != vnodeId) { + if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { writeDataToDisk(pTSBuf); shrinkBuffer(ptsData); - pBlockInfo = addOneVnodeInfo(pTSBuf, vnodeId); + pBlockInfo = addOneGroupInfo(pTSBuf, id); } else { - pBlockInfo = tsBufGetLastVnodeInfo(pTSBuf); + pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); } - assert(pBlockInfo->info.vnode == vnodeId); + assert(pBlockInfo->info.id == id); if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first @@ -464,23 +464,23 @@ void tsBufFlush(STSBuf* pTSBuf) { writeDataToDisk(pTSBuf); shrinkBuffer(&pTSBuf->tsData); - STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf); + STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); // update prev vnode length info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pBlockInfoEx->info); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); // save the ts order into header STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; STSBufUpdateHeader(pTSBuf, &header); fsync(fileno(pTSBuf->f)); } -static int32_t tsBufFindVnodeById(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { +static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) { int32_t j = -1; - for (int32_t i = 0; i < numOfVnodes; ++i) { - if (pVnodeInfoEx[i].info.vnode == vnodeId) { + for (int32_t i = 0; i < numOfGroups; ++i) { + if (pGroupInfoEx[i].info.id == id) { j = i; break; } @@ -490,7 +490,7 @@ static int32_t tsBufFindVnodeById(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t num } // todo opt performance by cache blocks info -static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int32_t blockIndex) { +static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } @@ -517,7 +517,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -544,14 +544,14 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo return -1; } -static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex) { - STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[vnodeIndex].info; +static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; if (pBlockInfo->numOfBlocks <= blockIndex) { assert(false); } STSCursor* pCur = &pTSBuf->cur; - if (pCur->vgroupIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || + if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; @@ -586,13 +586,13 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - pCur->vgroupIndex = vnodeIndex; + pCur->vgroupIndex = groupIndex; pCur->blockIndex = blockIndex; pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } -static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) { +static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) { if (offset < 0 || offset >= getDataStartOffset()) { return -1; } @@ -601,12 +601,12 @@ static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockIn return -1; } - fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f); + fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); return 0; } -STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { - int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); +STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { + int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); if (j == -1) { return NULL; } @@ -615,7 +615,7 @@ STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfVnode == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } @@ -631,7 +631,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { } bool tsBufNextPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL || pTSBuf->numOfVnodes == 0) { + if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { return false; } @@ -650,16 +650,16 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } else { // get the last timestamp record in the last block of the last vnode - assert(pTSBuf->numOfVnodes > 0); + assert(pTSBuf->numOfGroups > 0); - int32_t vnodeIndex = pTSBuf->numOfVnodes - 1; - pCur->vgroupIndex = vnodeIndex; + int32_t groupIndex = pTSBuf->numOfGroups - 1; + pCur->vgroupIndex = groupIndex; - int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode; - STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; + STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - tsBufGetBlock(pTSBuf, vnodeIndex, blockIndex); + tsBufGetBlock(pTSBuf, groupIndex, blockIndex); pCur->tsIndex = pTSBuf->block.numOfElem - 1; if (pTSBuf->block.numOfElem == 0) { @@ -678,12 +678,12 @@ bool tsBufNextPos(STSBuf* pTSBuf) { if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode; + int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - if ((pCur->vgroupIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) || + if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { pCur->vgroupIndex = -1; return false; @@ -719,7 +719,7 @@ void tsBufResetPos(STSBuf* pTSBuf) { } STSElem tsBufGetElem(STSBuf* pTSBuf) { - STSElem elem1 = {.vnode = -1}; + STSElem elem1 = {.id = -1}; if (pTSBuf == NULL) { return elem1; } @@ -731,7 +731,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { STSBlock* pBlock = &pTSBuf->block; - elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; + elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); elem1.tag = &pBlock->tag; @@ -742,34 +742,34 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { * current only support ts comp data from two vnode merge * @param pDestBuf * @param pSrcBuf - * @param vnodeId + * @param id * @return */ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { - if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfVnodes <= 0) { + if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { return 0; } - if (pDestBuf->numOfVnodes + pSrcBuf->numOfVnodes > TS_COMP_FILE_VNODE_MAX) { + if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { return -1; } // src can only have one vnode index - assert(pSrcBuf->numOfVnodes == 1); + assert(pSrcBuf->numOfGroups == 1); // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); // compared with the last vnode id - int32_t vnodeId = tsBufGetLastVnodeInfo((STSBuf*) pSrcBuf)->info.vnode; - if (vnodeId != tsBufGetLastVnodeInfo(pDestBuf)->info.vnode) { - int32_t oldSize = pDestBuf->numOfVnodes; - int32_t newSize = oldSize + pSrcBuf->numOfVnodes; + int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id; + if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { + int32_t oldSize = pDestBuf->numOfGroups; + int32_t newSize = oldSize + pSrcBuf->numOfGroups; if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - STSVnodeBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } @@ -778,23 +778,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } // directly copy the vnode index information - memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfVnodes * sizeof(STSVnodeBlockInfoEx)); + memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); // set the new offset value - for (int32_t i = 0; i < pSrcBuf->numOfVnodes; ++i) { - STSVnodeBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; + for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { + STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; - pBlockInfoEx->info.vnode = vnodeId; + pBlockInfoEx->info.id = id; } - pDestBuf->numOfVnodes = newSize; + pDestBuf->numOfGroups = newSize; } else { - STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pDestBuf); + STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); pBlockInfoEx->len += pSrcBuf->pData[0].len; pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; - pBlockInfoEx->info.vnode = vnodeId; + pBlockInfoEx->info.id = id; } int32_t r = fseek(pDestBuf->f, 0, SEEK_END); @@ -827,23 +827,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { assert(pDestBuf->fileSize == oldSize + size); -// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfVnode:%d, autoDelete:%d", pDestBuf, -// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfVnodes, pDestBuf->autoDelete); +// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfGroups:%d, autoDelete:%d", pDestBuf, +// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfGroups, pDestBuf->autoDelete); return 0; } -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t vnodeId) { +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { STSBuf* pTSBuf = tsBufCreate(true, order); - STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info); + STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); pBlockInfo->numOfBlocks = numOfBlocks; pBlockInfo->compLen = len; pBlockInfo->offset = getDataStartOffset(); - pBlockInfo->vnode = vnodeId; + pBlockInfo->id = id; // update prev vnode length info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); UNUSED(ret); @@ -855,7 +855,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; STSBufUpdateHeader(pTSBuf, &header); fsync(fileno(pTSBuf->f)); @@ -863,14 +863,14 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { - STSElem elem = {.vnode = -1}; +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag) { + STSElem elem = {.id = -1}; if (pTSBuf == NULL) { return elem; } - int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); if (j == -1) { return elem; } @@ -879,7 +879,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { // tsBufDisplay(pTSBuf); STSCursor* pCur = &pTSBuf->cur; - STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); if (blockIndex < 0) { @@ -935,7 +935,7 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) { void tsBufDisplay(STSBuf* pTSBuf) { printf("-------start of ts comp file-------\n"); - printf("number of vnode:%d\n", pTSBuf->numOfVnodes); + printf("number of vnode:%d\n", pTSBuf->numOfGroups); int32_t old = pTSBuf->cur.order; pTSBuf->cur.order = TSDB_ORDER_ASC; @@ -945,7 +945,7 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64Key, elem.ts); } } @@ -954,20 +954,20 @@ void tsBufDisplay(STSBuf* pTSBuf) { } static int32_t getDataStartOffset() { - return sizeof(STSBufFileHeader) + TS_COMP_FILE_VNODE_MAX * sizeof(STSVnodeBlockInfo); + return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo); } // update prev vnode length info in file -static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo) { - int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSVnodeBlockInfo); - doUpdateVnodeInfo(pTSBuf, offset, pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) { + int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo); + doUpdateGroupInfo(pTSBuf, offset, pBlockInfo); } static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { - const int32_t INITIAL_VNODEINFO_SIZE = 4; + const int32_t INITIAL_GROUPINFO_SIZE = 4; - pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx)); + pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; + pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -999,35 +999,35 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { return pTSBuf; } -int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf) { +int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return 0; } - return pTSBuf->numOfVnodes; + return pTSBuf->numOfGroups; } -void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { - int32_t size = tsBufGetNumOfVnodes(pTSBuf); +void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { + int32_t size = tsBufGetNumOfGroup(pTSBuf); if (num != NULL) { *num = size; } - *vnodeId = NULL; + *id = NULL; if (size == 0) { return; } - (*vnodeId) = malloc(tsBufGetNumOfVnodes(pTSBuf) * sizeof(int32_t)); + (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); for(int32_t i = 0; i < size; ++i) { - (*vnodeId)[i] = pTSBuf->pData[i].info.vnode; + (*id)[i] = pTSBuf->pData[i].info.id; } } -int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { - assert(vnodeIndex >= 0 && vnodeIndex < pTSBuf->numOfVnodes); - STSVnodeBlockInfo *pBlockInfo = &pTSBuf->pData[vnodeIndex].info; +int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { + assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); + STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info; *len = 0; *numOfBlocks = 0; @@ -1052,11 +1052,11 @@ int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeIndex, void* buf, in } STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { - STSElem el = {.vnode = -1}; + STSElem el = {.id = -1}; - for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) { - el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.vnode, pTag); - if (el.vnode == pTSBuf->pData[i].info.vnode) { + for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { + el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag); + if (el.id == pTSBuf->pData[i].info.id) { return el; } } @@ -1065,5 +1065,5 @@ STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { } bool tsBufIsValidElem(STSElem* pElem) { - return pElem->vnode >= 0; + return pElem->id >= 0; } From 9e7234b479a08c96f74c4061be9c3d6e603ea699 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 22:34:34 +0800 Subject: [PATCH 12/45] [TD-225] refactor codes. --- src/query/src/qAst.c | 48 +++++++------------------------------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 893105e44a..73744100ae 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -660,7 +660,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p * @param pSchema tag schemas * @param fp filter callback function */ -static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { +static UNUSED_FUNC void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { size_t size = taosArrayGetSize(pResult); SArray* array = taosArrayInit(size, POINTER_BYTES); @@ -749,48 +749,13 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } // recursive traverse left child branch + // The value of hasPK is always 0. uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); + //apply the hierarchical expression to every node in skiplist for find the qualified nodes + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - if (weight == 0 ) { - if (taosArrayGetSize(result) > 0 && pSkipList == NULL) { - /** - * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. - * Since no index presented, the filter operation is done by scan all elements in the result set. - * - * if the query is a high selectivity filter, only small portion of meters are retrieved. - */ - exprTreeTraverseImpl(pExpr, result, param); - } else { - /** - * apply the hierarchical expression to every node in skiplist for find the qualified nodes - */ - assert(taosArrayGetSize(result) == 0); - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } - - return; - } - - if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { - SArray* rLeft = taosArrayInit(10, POINTER_BYTES); - SArray* rRight = taosArrayInit(10, POINTER_BYTES); - - tExprTreeTraverse(pLeft, pSkipList, rLeft, param); - tExprTreeTraverse(pRight, pSkipList, rRight, param); - - if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS - intersect(rLeft, rRight, result); - } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - merge(rLeft, rRight, result); - } else { - assert(false); - } - - taosArrayDestroy(rLeft); - taosArrayDestroy(rRight); - return; - } - + #if 0 /* * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here * @@ -819,6 +784,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S * So, we do not set the skip list index as a parameter */ tExprTreeTraverse(pSecond, NULL, result, param); +#endif } void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, From ca5c1689640cd8e5610ae4001398127761b6a4e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 22:45:39 +0800 Subject: [PATCH 13/45] [TD-225] refactor. --- src/query/inc/qFill.h | 3 +-- src/query/src/qAst.c | 6 +----- src/query/src/qFill.c | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 329ea9a789..1b5aca77c4 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -49,7 +49,6 @@ typedef struct SFillInfo { int32_t numOfTags; // number of tags int32_t numOfCols; // number of columns, including the tags columns int32_t rowSize; // size of each row -// char ** pTags; // tags value for current interpolation SFillTagColInfo* pTags; // tags value for filling gap SInterval interval; char * prevValues; // previous row of data, to generate the interpolation results @@ -83,7 +82,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRo int32_t taosNumOfRemainRows(SFillInfo *pFillInfo); -int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); +int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity); diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 73744100ae..4f26a7489b 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -733,10 +733,6 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); param->setupInfoFn(pExpr, param->pExtInfo); - if (pSkipList == NULL) { - tArrayTraverse(pExpr, param->nodeFilterFn, result); - return; - } tQueryInfo *pQueryInfo = pExpr->_node.info; if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { @@ -748,10 +744,10 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S return; } - // recursive traverse left child branch // The value of hasPK is always 0. uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); + //apply the hierarchical expression to every node in skiplist for find the qualified nodes tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index 99fa3a8e0f..a219bd6abd 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -215,7 +215,7 @@ static double linearInterpolationImpl(double v1, double v2, double k1, double k2 return v1 + (v2 - v1) * (k - k1) / (k2 - k1); } -int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { +int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { switch (type) { case TSDB_DATA_TYPE_INT: { *(int32_t*)point->val = (int32_t)linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, (double)point1->key, @@ -343,7 +343,7 @@ static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* nu point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset}; point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes}; point = (SPoint){.key = pFillInfo->start, .val = val1}; - taosDoLinearInterpolation(type, &point1, &point2, &point); + taosGetLinearInterpolationVal(type, &point1, &point2, &point); } setTagsValue(pFillInfo, data, *num); From 848014d3e0f57565b535435045157dffee4f2704 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 22:54:49 +0800 Subject: [PATCH 14/45] [TD-225] refactor. --- src/query/src/qAst.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 4f26a7489b..847a8b8eb6 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -609,7 +609,7 @@ int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { /* * traverse the result and apply the function to each item to check if the item is qualified or not */ -static void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { +static UNUSED_FUNC void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL); // scan the result array list and check for each item in the list From a0880e0d2ee698b25bdd0de486cac1872b03d065 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 09:47:03 +0800 Subject: [PATCH 15/45] TD-1918 --- src/cq/src/cqMain.c | 2 +- src/cq/test/cqtest.c | 2 +- src/dnode/src/dnodeVWrite.c | 113 ++++++++++++------------------------ src/inc/tcq.h | 2 +- src/inc/tsync.h | 4 +- src/inc/twal.h | 4 +- src/inc/vnode.h | 22 +++++-- src/sync/src/syncMain.c | 2 +- src/sync/src/syncRestore.c | 4 +- src/vnode/inc/vnodeInt.h | 2 - src/vnode/src/vnodeMain.c | 3 +- src/vnode/src/vnodeWrite.c | 52 ++++++++--------- src/wal/src/walWrite.c | 4 +- src/wal/test/waltest.c | 2 +- 14 files changed, 95 insertions(+), 123 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 1a99a84b8e..9efa517ac3 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -334,7 +334,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pHead->version = 0; // write into vnode write queue - pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ); + pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL); free(buffer); } diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index 1daee644a7..e1114fc024 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -24,7 +24,7 @@ int64_t ver = 0; void *pCq = NULL; -int writeToQueue(void *pVnode, void *data, int type) { +int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { return 0; } diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 3a820f180c..aaf1098d40 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -31,14 +31,6 @@ typedef struct { pthread_t thread; // thread } SWriteWorker; -typedef struct { - SRspRet rspRet; - SRpcMsg rpcMsg; - int32_t processedCount; - int32_t code; - int32_t contLen; - void * pCont; -} SWriteMsg; typedef struct { int32_t max; // max number of workers @@ -86,39 +78,38 @@ void dnodeCleanupVWrite() { dInfo("dnode vwrite is closed"); } -void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg) { - char *pCont = pMsg->pCont; +void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) { + int32_t code; + char *pCont = pRpcMsg->pCont; - if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { + if (pRpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { SMsgDesc *pDesc = (SMsgDesc *)pCont; pDesc->numOfVnodes = htonl(pDesc->numOfVnodes); pCont += sizeof(SMsgDesc); } - SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = htonl(pHead->vgId); - pHead->contLen = htonl(pHead->contLen); + SMsgHead *pMsg = (SMsgHead *)pCont; + pMsg->vgId = htonl(pMsg->vgId); + pMsg->contLen = htonl(pMsg->contLen); - taos_queue queue = vnodeAcquireWqueue(pHead->vgId); - if (queue) { - // put message into queue - SWriteMsg *pWrite = taosAllocateQitem(sizeof(SWriteMsg)); - pWrite->rpcMsg = *pMsg; - pWrite->pCont = pCont; - pWrite->contLen = pHead->contLen; - - taosWriteQitem(queue, TAOS_QTYPE_RPC, pWrite); + void *pVnode = vnodeAcquire(pMsg->vgId); + if (pVnode == NULL) { + code = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { - SRpcMsg rpcRsp = { - .handle = pMsg->handle, - .pCont = NULL, - .contLen = 0, - .code = TSDB_CODE_VND_INVALID_VGROUP_ID, - .msgType = 0 - }; - rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); + SWalHead *pHead = (SWalHead *)(pCont - sizeof(SWalHead)); + pHead->msgType = pRpcMsg->msgType; + pHead->version = 0; + pHead->len = pMsg->contLen; + code = vnodeWriteToQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg); } + + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rpcRsp); + } + + vnodeRelease(pVnode); + rpcFreeCont(pRpcMsg->pCont); } void *dnodeAllocVWriteQueue(void *pVnode) { @@ -179,7 +170,7 @@ void dnodeFreeVWriteQueue(void *wqueue) { void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) { if (param == NULL) return; - SWriteMsg *pWrite = param; + SVWriteMsg *pWrite = param; if (code < 0) pWrite->code = code; int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); @@ -187,26 +178,22 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) { if (count <= 1) return; SRpcMsg rpcRsp = { - .handle = pWrite->rpcMsg.handle, + .handle = pWrite->rpcHandle, .pCont = pWrite->rspRet.rsp, .contLen = pWrite->rspRet.len, .code = pWrite->code, }; rpcSendResponse(&rpcRsp); - rpcFreeCont(pWrite->rpcMsg.pCont); taosFreeQitem(pWrite); vnodeRelease(pVnode); } static void *dnodeProcessWriteQueue(void *param) { - SWriteWorker *pWorker = (SWriteWorker *)param; - SWriteMsg * pWrite; - SWalHead * pHead; - SRspRet * pRspRet; + SWriteWorker *pWorker = param; + SVWriteMsg * pWrite; void * pVnode; - void * pItem; int32_t numOfMsgs; int32_t qtype; @@ -220,36 +207,14 @@ static void *dnodeProcessWriteQueue(void *param) { } for (int32_t i = 0; i < numOfMsgs; ++i) { - pWrite = NULL; - pRspRet = NULL; - taosGetQitem(pWorker->qall, &qtype, &pItem); - if (qtype == TAOS_QTYPE_RPC) { - pWrite = pItem; - pRspRet = &pWrite->rspRet; - pHead = (SWalHead *)((char *)pWrite->pCont - sizeof(SWalHead)); - pHead->msgType = pWrite->rpcMsg.msgType; - pHead->version = 0; - pHead->len = pWrite->contLen; - dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType]); - } else if (qtype == TAOS_QTYPE_CQ) { - pHead = (SWalHead *)((char *)pItem + sizeof(SSyncHead)); - dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], - pHead->version); - } else { - pHead = pItem; - dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], - pHead->version); - } + taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); + dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%d version:%" PRIu64, pWrite->rpcAhandle, pWrite, + taosMsg[pWrite->pHead->msgType], qtype, pWrite->pHead->version); - int32_t code = vnodeProcessWrite(pVnode, qtype, pHead, pRspRet); - dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType], - pHead->version, tstrerror(code)); + pWrite->code = vnodeProcessWrite(pVnode, qtype, pWrite); + if (pWrite->code <= 0) pWrite->processedCount = 1; - if (pWrite) { - pWrite->rpcMsg.code = code; - if (code <= 0) pWrite->processedCount = 1; - } + dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code)); } walFsync(vnodeGetWal(pVnode)); @@ -257,17 +222,15 @@ static void *dnodeProcessWriteQueue(void *param) { // browse all items, and process them one by one taosResetQitems(pWorker->qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(pWorker->qall, &qtype, &pItem); + taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); if (qtype == TAOS_QTYPE_RPC) { - pWrite = pItem; - dnodeSendRpcVWriteRsp(pVnode, pItem, pWrite->rpcMsg.code); + dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); } else if (qtype == TAOS_QTYPE_FWD) { - pHead = pItem; - vnodeConfirmForward(pVnode, pHead->version, 0); - taosFreeQitem(pItem); + vnodeConfirmForward(pVnode, pWrite->pHead->version, 0); + taosFreeQitem(pWrite); vnodeRelease(pVnode); } else { - taosFreeQitem(pItem); + taosFreeQitem(pWrite); vnodeRelease(pVnode); } } diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 32b75674c3..4a23695a1a 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -21,7 +21,7 @@ extern "C" { #include "tdataformat.h" -typedef int (*FCqWrite)(void *ahandle, void *pHead, int type); +typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); typedef struct { int vgId; diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 671adefab8..ca11e0ae0b 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -70,8 +70,8 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); -// when a forward pkt is received, call this to handle data -typedef int (*FWriteToCache)(void *ahandle, void *pHead, int type); +// when a forward pkt is received, call this to handle data +typedef int32_t (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); // when forward is confirmed by peer, master call this API to notify app typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); diff --git a/src/inc/twal.h b/src/inc/twal.h index 3a229ed835..d9b5fb26ff 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -43,8 +43,8 @@ typedef struct { int8_t keep; // keep the wal file when closed } SWalCfg; -typedef void* twalh; // WAL HANDLE -typedef int (*FWalWrite)(void *ahandle, void *pHead, int type); +typedef void * twalh; // WAL HANDLE +typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg); int32_t walInit(); void walCleanUp(); diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 8c387065cc..670f4f599c 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "twal.h" + typedef enum _VN_STATUS { TAOS_VN_STATUS_INIT, TAOS_VN_STATUS_READY, @@ -29,9 +31,9 @@ typedef enum _VN_STATUS { } EVnStatus; typedef struct { - int len; - void *rsp; - void *qhandle; //used by query and retrieve msg + int32_t len; + void * rsp; + void * qhandle; // used by query and retrieve msg } SRspRet; typedef struct { @@ -41,6 +43,16 @@ typedef struct { SRpcMsg rpcMsg; } SReadMsg; +typedef struct { + int32_t code; + int32_t processedCount; + void * rpcHandle; + void * rpcAhandle; + SRspRet rspRet; + char reserveForSync[16]; + SWalHead pHead[]; +} SVWriteMsg; + extern char *vnodeStatus[]; int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); @@ -55,7 +67,9 @@ void* vnodeAcquireWqueue(int32_t vgId); // add recCount, get write queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); -int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item); + +int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg); +int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite); int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 6f5e3be8ab..a504ded657 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -863,7 +863,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD); + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 19a5d3ba41..9216567bc5 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -154,7 +154,7 @@ static int syncRestoreWal(SSyncPeer *pPeer) { if (ret < 0) break; sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version); - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL); + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL); } if (code < 0) { @@ -169,7 +169,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)offset; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD); + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); offset += pHead->len + sizeof(SWalHead); return offset; diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 169334c611..935da54367 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -61,8 +61,6 @@ typedef struct { char db[TSDB_DB_NAME_LEN]; } SVnodeObj; -int vnodeWriteToQueue(void *param, void *pHead, int type); -int vnodeWriteCqMsgToQueue(void *param, void *pHead, int type); void vnodeInitWriteFp(void); void vnodeInitReadFp(void); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e206933116..7cbbf0feb8 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -266,7 +266,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { strcpy(cqCfg.pass, tsInternalPass); strcpy(cqCfg.db, pVnode->db); cqCfg.vgId = vnode; - cqCfg.cqWrite = vnodeWriteCqMsgToQueue; + cqCfg.cqWrite = vnodeWriteToQueue; pVnode->cq = cqOpen(pVnode, &cqCfg); if (pVnode->cq == NULL) { vnodeCleanUp(pVnode); @@ -365,6 +365,7 @@ int32_t vnodeClose(int32_t vgId) { } void vnodeRelease(void *pVnodeRaw) { + if (pVnodeRaw == NULL) return; SVnodeObj *pVnode = pVnodeRaw; int32_t vgId = pVnode->vgId; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index c2771c73c6..4432f98db0 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -46,10 +46,10 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } -int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { +int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite) { int32_t code = 0; - SVnodeObj *pVnode = (SVnodeObj *)param1; - SWalHead * pHead = param2; + SVnodeObj *pVnode = param; + SWalHead * pHead = pWrite->pHead; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]); @@ -80,7 +80,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { // forward to peers, even it is WAL/FWD, it shall be called to update version in sync int32_t syncCode = 0; - syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype); + syncCode = syncForwardToPeer(pVnode->sync, pHead, &pWrite->rspRet, qtype); if (syncCode < 0) return syncCode; // write into WAL @@ -90,7 +90,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { pVnode->version = pHead->version; // write data locally - code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); + code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, &pWrite->rspRet); if (code < 0) return code; return syncCode; @@ -204,35 +204,31 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } -int vnodeWriteCqMsgToQueue(void *param, void *data, int type) { - SVnodeObj *pVnode = param; - SWalHead * pHead = data; +int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg) { + SVnodeObj *pVnode = vparam; + SWalHead * pHead = wparam; - int size = sizeof(SWalHead) + pHead->len; - SSyncHead *pSync = (SSyncHead*) taosAllocateQitem(size + sizeof(SSyncHead)); - SWalHead *pWal = (SWalHead *)(pSync + 1); - memcpy(pWal, pHead, size); + if (qtype == TAOS_QTYPE_RPC && vnodeCheckWrite(pVnode) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } - atomic_add_fetch_32(&pVnode->refCount, 1); - vTrace("CQ: vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; + SVWriteMsg *pWrite = taosAllocateQitem(size); + if (pWrite == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } - taosWriteQitem(pVnode->wqueue, type, pSync); + if (pMsg != NULL) { + SRpcMsg *pRpcMsg = pMsg; + pWrite->rpcHandle = pRpcMsg->handle; + pWrite->rpcAhandle = pRpcMsg->ahandle; + } - return 0; -} - -int vnodeWriteToQueue(void *param, void *data, int type) { - SVnodeObj *pVnode = param; - SWalHead * pHead = data; - - int size = sizeof(SWalHead) + pHead->len; - SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); - memcpy(pWal, pHead, size); + memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len); atomic_add_fetch_32(&pVnode->refCount, 1); vTrace("vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - taosWriteQitem(pVnode->wqueue, type, pWal); - - return 0; + taosWriteQitem(pVnode->wqueue, qtype, pWrite); + return TSDB_CODE_SUCCESS; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e57cb0e042..d85f740597 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -122,7 +122,7 @@ void walFsync(void *handle) { } } -int32_t walRestore(void *handle, void *pVnode, int32_t (*writeFp)(void *, void *, int32_t)) { +int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { if (handle == NULL) return -1; SWal * pWal = handle; @@ -307,7 +307,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch if (pWal->keep) pWal->version = pHead->version; - (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL); + (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); } tclose(fd); diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 186f2ef5ff..14e439c072 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -23,7 +23,7 @@ int64_t ver = 0; void *pWal = NULL; -int writeToQueue(void *pVnode, void *data, int type) { +int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { // do nothing SWalHead *pHead = data; From 16132ff943a83253862f14d6d95de62e90f62ce8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 09:57:25 +0800 Subject: [PATCH 16/45] TD-1918 --- src/inc/vnode.h | 2 -- src/mnode/src/mnodeSdb.c | 14 +++++++------- src/vnode/src/vnodeMain.c | 15 --------------- src/vnode/src/vnodeWrite.c | 5 +++-- 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 670f4f599c..8b3b4b6ed0 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -63,11 +63,9 @@ int32_t vnodeClose(int32_t vgId); void* vnodeAcquire(int32_t vgId); // add refcount void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue -void* vnodeAcquireWqueue(int32_t vgId); // add recCount, get write queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); - int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg); int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite); int32_t vnodeCheckWrite(void *pVnode); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 8fb0b33060..fc986521e6 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -98,8 +98,8 @@ static taos_qall tsSdbWriteQall; static taos_queue tsSdbWriteQueue; static SSdbWriteWorkerPool tsSdbPool; -static int sdbWrite(void *param, void *data, int type); -static int sdbWriteToQueue(void *param, void *data, int type); +static int32_t sdbWrite(void *param, void *data, int32_t type, void *pMsg); +static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg); static void * sdbWorkerFp(void *param); static int32_t sdbInitWriteWorker(); static void sdbCleanupWriteWorker(); @@ -575,7 +575,7 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int sdbWrite(void *param, void *data, int type) { +static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { SSdbOper *pOper = param; SWalHead *pHead = data; int32_t tableId = pHead->msgType / 10; @@ -1040,13 +1040,13 @@ void sdbFreeWritequeue() { tsSdbWriteQueue = NULL; } -int sdbWriteToQueue(void *param, void *data, int type) { +int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { SWalHead *pHead = data; - int size = sizeof(SWalHead) + pHead->len; + int32_t size = sizeof(SWalHead) + pHead->len; SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); memcpy(pWal, pHead, size); - taosWriteQitem(tsSdbWriteQueue, type, pWal); + taosWriteQitem(tsSdbWriteQueue, qtype, pWal); return 0; } @@ -1081,7 +1081,7 @@ static void *sdbWorkerFp(void *param) { pOper = NULL; } - int32_t code = sdbWrite(pOper, pHead, type); + int32_t code = sdbWrite(pOper, pHead, type, NULL); if (code > 0) code = 0; if (pOper) { pOper->retCode = code; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 7cbbf0feb8..49b7e4e712 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -483,21 +483,6 @@ void *vnodeAcquireRqueue(int32_t vgId) { return pVnode->rqueue; } -void *vnodeAcquireWqueue(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) return NULL; - - int32_t code = vnodeCheckWrite(pVnode); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - vInfo("vgId:%d, can not provide write service, status is %s", vgId, vnodeStatus[pVnode->status]); - vnodeRelease(pVnode); - return NULL; - } - - return pVnode->wqueue; -} - void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 4432f98db0..0d521b4d2e 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -208,8 +208,9 @@ int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg) SVnodeObj *pVnode = vparam; SWalHead * pHead = wparam; - if (qtype == TAOS_QTYPE_RPC && vnodeCheckWrite(pVnode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_VND_INVALID_VGROUP_ID; + if (qtype == TAOS_QTYPE_RPC) { + int32_t code = vnodeCheckWrite(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; } int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; From 3dc6e228ee255fd02de0d4040804cceec0c8eea3 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 5 Nov 2020 10:25:36 +0800 Subject: [PATCH 17/45] fix join case error --- tests/pytest/query/queryJoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index a5e3ab21b3..5ad49a265e 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -106,7 +106,7 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.pid") - tdSql.checkRows(3) + tdSql.checkRows(6) tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.checkRows(6) From 199040986ffda9abc8be622dc48cb822865ba20f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 5 Nov 2020 11:21:44 +0800 Subject: [PATCH 18/45] fix test case for interval --- tests/pytest/query/queryInterval.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 9922201604..ec3255ba7f 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -35,10 +35,11 @@ class TDTestCase: % (self.ts, self.ts + 2000000000, self.ts + 4000000000, self.ts + 5000000000, self.ts + 7000000000)) tdSql.query("select avg(voltage) from st interval(1n)") - tdSql.checkRows(3) - tdSql.checkData(0, 1, 221.4) - tdSql.checkData(1, 1, 227.0) - tdSql.checkData(2, 1, 222.0) + tdSql.checkRows(4) + tdSql.checkData(0, 1, 220.0) + tdSql.checkData(1, 1, 222.33333333333334) + tdSql.checkData(2, 1, 227.0) + tdSql.checkData(3, 1, 222.0) tdSql.query("select avg(voltage) from st interval(1n, 15d)") tdSql.checkRows(4) From 62b857e4f8da9f4d0b8f2ae6d1a281e1051d6b25 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 11:31:40 +0800 Subject: [PATCH 19/45] change --- .../com/taosdata/jdbc/TSDBDriverTest.java | 189 ++++++++---------- 1 file changed, 84 insertions(+), 105 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 8718c3e7da..627daf76ac 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -13,7 +13,7 @@ import static org.junit.Assert.*; public class TSDBDriverTest { - private static String[] validURLs = { + private static final String[] validURLs = { "jdbc:TAOS://localhost:0", "jdbc:TAOS://localhost", "jdbc:TAOS://localhost:6030/test", @@ -27,45 +27,99 @@ public class TSDBDriverTest { "jdbc:TAOS://:/test", "jdbc:TAOS://localhost:0/?user=root&password=taosdata" }; - private static boolean islibLoaded; + private static boolean islibLoaded = false; private static boolean isTaosdActived; @BeforeClass public static void before() { String osName = System.getProperty("os.name").toLowerCase(); - if (!osName.equals("linux") && !osName.equals("windows")) { - islibLoaded = false; + if (!osName.equals("linux")) return; - } + // try to load taos lib try { System.loadLibrary("taos"); islibLoaded = true; } catch (UnsatisfiedLinkError error) { System.out.println("load tdengine lib failed."); - islibLoaded = false; + error.printStackTrace(); } - + // check taosd is activated try { - if (osName.equals("linux")) { - String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; - Process exec = Runtime.getRuntime().exec(cmd); - BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); - int lineCnt = 0; - while (reader.readLine() != null) { - lineCnt++; - } - if (lineCnt > 0) - isTaosdActived = true; - else - isTaosdActived = false; - } else { - isTaosdActived = false; + String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; + Process exec = Runtime.getRuntime().exec(cmd); + BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); + int lineCnt = 0; + while (reader.readLine() != null) { + lineCnt++; } + if (lineCnt > 0) + isTaosdActived = true; } catch (IOException e) { - isTaosdActived = false; + e.printStackTrace(); } } + + @Test + public void testConnectWithJdbcURL() { + final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; + try { + if (islibLoaded && isTaosdActived) { + Connection conn = DriverManager.getConnection(url); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("failure - should not throw Exception"); + } + } + + @Test + public void testConnectWithProperties() { + final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + try { + if (islibLoaded && isTaosdActived) { + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("failure - should not throw Exception"); + } + } + + @Test + public void testConnectWithConfigFile() { + String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + try { + if (islibLoaded && isTaosdActived) { + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("failure - should not throw Exception"); + } + } + + @Test(expected = SQLException.class) + public void testAcceptsURL() throws SQLException { + Driver driver = new TSDBDriver(); + for (String url : validURLs) { + assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); + } + driver.acceptsURL(null); + fail("acceptsURL throws exception when parameter is null"); + } + @Test public void testParseURL() { TSDBDriver driver = new TSDBDriver(); @@ -85,7 +139,7 @@ public class TSDBDriverTest { actual = driver.parseURL(url, config); assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host")); assertEquals("failure - port should be 0", "0", actual.get("port")); - assertEquals("failure - dbname should be null", null, actual.get("dbname")); + assertNull("failure - dbname should be null", actual.get("dbname")); url = "jdbc:TAOS://127.0.0.1:0/db"; config = new Properties(); @@ -101,79 +155,9 @@ public class TSDBDriverTest { actual = driver.parseURL(url, config); assertEquals("failure - user should be root", "root", actual.getProperty("user")); assertEquals("failure - password should be taosdata", "taosdata", actual.getProperty("password")); - assertEquals("failure - host should be null", null, actual.getProperty("host")); - assertEquals("failure - port should be null", null, actual.getProperty("port")); - assertEquals("failure - dbname should be null", null, actual.getProperty("dbname")); - } - - @Test - public void testConnectWithJdbcURL() { - final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; - try { - if (islibLoaded) { - Connection conn = DriverManager.getConnection(url); - assertNotNull("failure - connection should not be null", conn); - } - } catch (SQLException e) { - e.printStackTrace(); - if (!isTaosdActived) - assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); - else - fail("failure - should not throw Exception"); - } - } - - @Test - public void testConnectWithProperties() { - final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - try { - if (islibLoaded) { - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } - } catch (SQLException e) { - if (!isTaosdActived) { - assertEquals("failure - should throw SQLException", SQLException.class, e.getClass()); - } else { - fail("failure - should not throw Exception"); - } - } - } - - @Test - public void testConnectWithConfigFile() { - String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - try { - if (islibLoaded) { - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - System.out.println(conn); - assertNotNull("failure - connection should not be null", conn); - } - } catch (SQLException e) { - e.printStackTrace(); - if (!isTaosdActived) - assertEquals("failure - should throw SQLException", "TDengine Error: Unable to establish connection", e.getMessage()); - else - fail("failure - should not throw Exception"); - } - } - - @Test(expected = SQLException.class) - public void testAcceptsURL() throws SQLException { - Driver driver = new TSDBDriver(); - for (String url : validURLs) { - assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); - } - new TSDBDriver().acceptsURL(null); - fail("acceptsURL throws exception when parameter is null"); + assertNull("failure - host should be null", actual.getProperty("host")); + assertNull("failure - port should be null", actual.getProperty("port")); + assertNull("failure - dbname should be null", actual.getProperty("dbname")); } @Test @@ -181,9 +165,6 @@ public class TSDBDriverTest { Driver driver = new TSDBDriver(); final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, connProps); for (DriverPropertyInfo info : propertyInfo) { if (info.name.equals(TSDBDriver.PROPERTY_KEY_HOST)) @@ -200,20 +181,18 @@ public class TSDBDriverTest { } @Test - public void testGetMajorVersion() throws SQLException { - Driver driver = new TSDBDriver(); - assertEquals("failure - getMajorVersion should be 2", 2, driver.getMajorVersion()); + public void testGetMajorVersion() { + assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion()); } @Test public void testGetMinorVersion() { - Driver driver = new TSDBDriver(); - assertEquals("failure - getMinorVersion should be 0", 0, driver.getMinorVersion()); + assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion()); } @Test public void testJdbcCompliant() { -// assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant()); + assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant()); } @Test From b8d48e8ea0dcefdc51fcec6a32d19b0a96f66562 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 5 Nov 2020 11:36:46 +0800 Subject: [PATCH 20/45] [TD-1939]: possible crash in wildcard query --- src/tsdb/src/tsdbRead.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 90e16aee53..4916a2d306 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2311,7 +2311,11 @@ void filterPrepare(void* expr, void* param) { if (pInfo->optr == TSDB_RELATION_IN) { pInfo->q = (char*) pCond->arr; } else { - pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. + uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE; + if (size < (uint32_t)pSchema->bytes) { + size = pSchema->bytes; + } + pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. tVariantDump(pCond, pInfo->q, pSchema->type, true); } } From faa8ff7cc24740859f231d1a7ba29f594d7abc14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 5 Nov 2020 03:43:26 +0000 Subject: [PATCH 21/45] [TD-1937]:client crash on percentile --- src/client/src/tscLocalMerge.c | 3 +++ src/query/src/qHistogram.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index c67edf5b5a..f49c4da0cb 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -97,6 +97,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64Key = pQueryInfo->order.orderColId; + } else if (functionId == TSDB_FUNC_APERCT) { + pCtx->param[0].i64Key = pExpr->param[0].i64Key; + pCtx->param[0].nType = pExpr->param[0].nType; } pCtx->interBufBytes = pExpr->interBytes; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 703ee2c521..35e5906d1f 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { (*pHisto)->numOfEntries += 1; } } else { /* insert a new slot */ - if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) { + if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { assert((*pHisto)->elems[idx - 1].val <= val); } @@ -661,4 +661,4 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 free(pHistoBins); return pResHistogram; -} \ No newline at end of file +} From 6874de8ebe5862e70588c3d81dc5fd4cd79c4add Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 05:08:04 +0000 Subject: [PATCH 22/45] TD-1842 --- src/dnode/src/dnodeVWrite.c | 2 +- src/inc/vnode.h | 4 ++-- src/vnode/src/vnodeMain.c | 2 +- src/vnode/src/vnodeWrite.c | 19 ++++++++++--------- src/wal/src/walWrite.c | 3 +-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index aaf1098d40..9b58d7bb61 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -211,7 +211,7 @@ static void *dnodeProcessWriteQueue(void *param) { dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%d version:%" PRIu64, pWrite->rpcAhandle, pWrite, taosMsg[pWrite->pHead->msgType], qtype, pWrite->pHead->version); - pWrite->code = vnodeProcessWrite(pVnode, qtype, pWrite); + pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); if (pWrite->code <= 0) pWrite->processedCount = 1; dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code)); diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 8b3b4b6ed0..09273d3907 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -66,8 +66,8 @@ void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); -int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg); -int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite); +int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); +int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 49b7e4e712..a13addb27c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -305,7 +305,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { return terrno; } - walRestore(pVnode->wal, pVnode, vnodeWriteToQueue); + walRestore(pVnode->wal, pVnode, vnodeProcessWrite); if (pVnode->version == 0) { pVnode->version = walGetVersion(pVnode->wal); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 0d521b4d2e..417448423a 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -46,10 +46,11 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } -int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite) { - int32_t code = 0; - SVnodeObj *pVnode = param; - SWalHead * pHead = pWrite->pHead; +int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { + int32_t code = 0; + SVnodeObj * pVnode = vparam; + SWalHead * pHead = wparam; + SRspRet * pRspRet = rparam; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]); @@ -80,7 +81,7 @@ int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite) { // forward to peers, even it is WAL/FWD, it shall be called to update version in sync int32_t syncCode = 0; - syncCode = syncForwardToPeer(pVnode->sync, pHead, &pWrite->rspRet, qtype); + syncCode = syncForwardToPeer(pVnode->sync, pHead, pRspRet, qtype); if (syncCode < 0) return syncCode; // write into WAL @@ -90,7 +91,7 @@ int32_t vnodeProcessWrite(void *param, int32_t qtype, SVWriteMsg *pWrite) { pVnode->version = pHead->version; // write data locally - code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, &pWrite->rspRet); + code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet); if (code < 0) return code; return syncCode; @@ -204,7 +205,7 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } -int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg) { +int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; SWalHead * pHead = wparam; @@ -219,8 +220,8 @@ int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *pMsg) return TSDB_CODE_VND_OUT_OF_MEMORY; } - if (pMsg != NULL) { - SRpcMsg *pRpcMsg = pMsg; + if (rparam != NULL) { + SRpcMsg *pRpcMsg = rparam; pWrite->rpcHandle = pRpcMsg->handle; pWrite->rpcAhandle = pRpcMsg->ahandle; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index d85f740597..470607e965 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -305,8 +305,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch wTrace("vgId:%d, fileId:%" PRId64 ", restore wal ver:%" PRIu64 ", head ver:%" PRIu64 " len:%d", pWal->vgId, fileId, pWal->version, pHead->version, pHead->len); - if (pWal->keep) pWal->version = pHead->version; - + pWal->version = pHead->version; (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); } From 8e5835ae3786f3742f8ab25de49ec339dd0c1c3a Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 5 Nov 2020 13:42:18 +0800 Subject: [PATCH 23/45] [TD-1945] do not install libtaos.so inot lib64 when lib64 not exist in som so --- packaging/tools/post.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 569f316ff3..00705fad77 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -81,8 +81,10 @@ function install_lib() { ${csudo} ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi } function install_bin() { From b49b361fdf302d9f6967bc63329c17c7d56e1c01 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 13:47:55 +0800 Subject: [PATCH 24/45] change --- .../jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java | 8 ++++++++ .../src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index cd45048a77..63c42ca399 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -249,6 +249,7 @@ public class TSDBDriver implements java.sql.Driver { return null; // parse properties + String urlForMeta = url; int beginningOfSlashes = url.indexOf("//"); int index = url.indexOf("?"); if (index != -1) { @@ -271,6 +272,10 @@ public class TSDBDriver implements java.sql.Driver { } } } + // parse Product Name + String dbProductName = url.substring(0, beginningOfSlashes); + dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1); + dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); // parse dbname url = url.substring(beginningOfSlashes + 2); int indexOfSlash = url.indexOf("/"); @@ -291,6 +296,9 @@ public class TSDBDriver implements java.sql.Driver { if (url != null && url.length() > 0 && url.trim().length() > 0) { urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); } + + this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER)); + /* String urlForMeta = url; String dbProductName = url.substring(url.indexOf(":") + 1); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 627daf76ac..b4a7af127a 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -57,6 +57,12 @@ public class TSDBDriverTest { } catch (IOException e) { e.printStackTrace(); } + + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } } From c62dec736af3a9ab0ff95401f64b35613a1b9661 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 13:56:32 +0800 Subject: [PATCH 25/45] change --- .../com/taosdata/jdbc/TSDBDriverTest.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index b4a7af127a..f821266602 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -1,5 +1,6 @@ package com.taosdata.jdbc; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; @@ -30,6 +31,8 @@ public class TSDBDriverTest { private static boolean islibLoaded = false; private static boolean isTaosdActived; + private Connection conn; + @BeforeClass public static void before() { String osName = System.getProperty("os.name").toLowerCase(); @@ -71,7 +74,7 @@ public class TSDBDriverTest { final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; try { if (islibLoaded && isTaosdActived) { - Connection conn = DriverManager.getConnection(url); + conn = DriverManager.getConnection(url); assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { @@ -89,7 +92,7 @@ public class TSDBDriverTest { connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { if (islibLoaded && isTaosdActived) { - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + conn = DriverManager.getConnection(jdbcUrl, connProps); assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { @@ -107,7 +110,7 @@ public class TSDBDriverTest { connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { if (islibLoaded && isTaosdActived) { - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + conn = DriverManager.getConnection(jdbcUrl, connProps); assertNotNull("failure - connection should not be null", conn); } } catch (SQLException e) { @@ -205,4 +208,22 @@ public class TSDBDriverTest { public void testGetParentLogger() throws SQLFeatureNotSupportedException { assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); } + + @After + public void after() { + Statement statement = null; + try { + statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery("show databases"); + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + System.out.print(metaData.getColumnLabel(i) + " : " + resultSet.getString(i) + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } } \ No newline at end of file From 9400ab65cc5ccd7edefd4d4a4e223c82c158d6d8 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 5 Nov 2020 13:58:12 +0800 Subject: [PATCH 26/45] update TAOS SQL --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 191af56bbd..905d3b2cd7 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -1016,9 +1016,9 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER ``` ## TAOS SQL 边界限制 -- 数据库名最大长度为33 -- 表名最大长度为193，每行数据最大长度16k个字符 -- 列名最大长度为65，最多允许1024列，最少需要2列，第一列必须是时间戳 +- 数据库名最大长度为32 +- 表名最大长度为192，每行数据最大长度16k个字符 +- 列名最大长度为64，最多允许1024列，最少需要2列，第一列必须是时间戳 - 标签最多允许128个，可以0个，标签总长度不超过16k个字符 - SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 From ce2287ef7fbf99c578e7caa9642f7cc7534bcb50 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 13:58:28 +0800 Subject: [PATCH 27/45] change --- .../src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index f821266602..dbfc68c100 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -224,6 +224,15 @@ public class TSDBDriverTest { } } catch (SQLException e) { e.printStackTrace(); + } finally { + try { + if (statement != null) + statement.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } } } } \ No newline at end of file From be722bdf48b314ce65237765e5b5aad2d81c97c7 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 14:00:40 +0800 Subject: [PATCH 28/45] change --- .../jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index dbfc68c100..cdda5ea894 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -113,6 +113,7 @@ public class TSDBDriverTest { conn = DriverManager.getConnection(jdbcUrl, connProps); assertNotNull("failure - connection should not be null", conn); } + printRs(); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -209,8 +210,7 @@ public class TSDBDriverTest { assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); } - @After - public void after() { + public void printRs() { Statement statement = null; try { statement = conn.createStatement(); From 07a39d381316897bb480d917a0868da1ebf2959e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 Nov 2020 14:00:03 +0800 Subject: [PATCH 29/45] [TD-1946] : fix compile issues for VS2019 16.7 on Windows --- src/client/src/tscSubquery.c | 2 +- src/os/src/windows/wAtomic.c | 2 +- src/os/src/windows/wFile.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index de0d899f8c..5139f7bd80 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2366,7 +2366,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - pRes->numOfCols = (int32_t)numOfExprs; + pRes->numOfCols = (int16_t)numOfExprs; pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); pRes->buffer = calloc(numOfExprs, POINTER_BYTES); diff --git a/src/os/src/windows/wAtomic.c b/src/os/src/windows/wAtomic.c index a025cb8f0e..b645893030 100644 --- a/src/os/src/windows/wAtomic.c +++ b/src/os/src/windows/wAtomic.c @@ -44,7 +44,7 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { //#ifdef _WIN64 - return _InterlockedExchangeAdd64(ptr, val) + val; + return InterlockedExchangeAdd64(ptr, val) + val; //#else // return _InterlockedExchangeAdd(ptr, val) + val; //#endif diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 734ed9916d..2204135ae6 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -65,12 +65,12 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co int64_t remain = count - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); + size_t rlen = fread(buffer, 1, (size_t) remain, in_file); if (rlen <= 0) { return writeLen; } else { - fwrite(buffer, 1, remain, out_file); + fwrite(buffer, 1, (size_t) remain, out_file); writeLen += remain; } } From 7cbbc0733e0a9e6a8e5af4397aaafcfa9a6598dc Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 14:22:19 +0800 Subject: [PATCH 30/45] change --- .../src/main/java/com/taosdata/jdbc/TSDBConnection.java | 3 ++- .../main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java | 2 +- .../src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 8 +++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index ac0e4eb84a..57230ee65b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -197,12 +197,13 @@ public class TSDBConnection implements Connection { } public SQLWarning getWarnings() throws SQLException { + //todo: implement getWarnings according to the warning messages returned from TDengine throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public void clearWarnings() throws SQLException { // left blank to support HikariCP connection - //todo: implement getWarnings according to the warning messages returned from TDengine + //todo: implement clearWarnings according to the warning messages returned from TDengine } public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java index c1d9d2af8e..f4dee67adf 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java @@ -96,7 +96,7 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { } public int getDriverMajorVersion() { - return 0; + return 2; } public int getDriverMinorVersion() { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index cdda5ea894..d254b242af 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -8,6 +8,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.sql.*; +import java.util.Enumeration; import java.util.Properties; import static org.junit.Assert.*; @@ -68,7 +69,6 @@ public class TSDBDriverTest { } } - @Test public void testConnectWithJdbcURL() { final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; @@ -114,6 +114,12 @@ public class TSDBDriverTest { assertNotNull("failure - connection should not be null", conn); } printRs(); + Properties clientInfo = conn.getClientInfo(); + Enumeration propertyNames = clientInfo.propertyNames(); + while (propertyNames.hasMoreElements()) { + String name = (String) propertyNames.nextElement(); + System.out.println(name + " : " + clientInfo.getProperty(name)); + } } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); From bc94bec5c3be69e75c2a34ebc00ba2f5dc0e3192 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 14:26:16 +0800 Subject: [PATCH 31/45] change --- .../jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 63c42ca399..d5f8510320 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -168,6 +168,13 @@ public class TSDBDriver implements java.sql.Driver { } try { + + Enumeration propertyNames = props.propertyNames(); + while (propertyNames.hasMoreElements()) { + Object name = propertyNames.nextElement(); + System.out.println(name + " : " + props.get(name)); + } + TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET), From 5e4b45728055f8e2e179810a07d120bd2c3d2eb6 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 14:40:47 +0800 Subject: [PATCH 32/45] change --- .../src/main/java/com/taosdata/jdbc/TSDBConnection.java | 1 - .../jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java | 7 ------- .../src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 7 ------- 3 files changed, 15 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 57230ee65b..294544ed47 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -52,7 +52,6 @@ public class TSDBConnection implements Connection { public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { this.dbMetaData = meta; - connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index d5f8510320..63c42ca399 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -168,13 +168,6 @@ public class TSDBDriver implements java.sql.Driver { } try { - - Enumeration propertyNames = props.propertyNames(); - while (propertyNames.hasMoreElements()) { - Object name = propertyNames.nextElement(); - System.out.println(name + " : " + props.get(name)); - } - TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET), diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index d254b242af..e7425a1569 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -113,13 +113,6 @@ public class TSDBDriverTest { conn = DriverManager.getConnection(jdbcUrl, connProps); assertNotNull("failure - connection should not be null", conn); } - printRs(); - Properties clientInfo = conn.getClientInfo(); - Enumeration propertyNames = clientInfo.propertyNames(); - while (propertyNames.hasMoreElements()) { - String name = (String) propertyNames.nextElement(); - System.out.println(name + " : " + clientInfo.getProperty(name)); - } } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); From 42add31acd4355510e48b8404abe841fff4d62d7 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 14:44:01 +0800 Subject: [PATCH 33/45] change --- .../com/taosdata/jdbc/TSDBDriverTest.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index e7425a1569..8adcdefb29 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc; -import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; @@ -8,7 +7,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.sql.*; -import java.util.Enumeration; import java.util.Properties; import static org.junit.Assert.*; @@ -209,29 +207,4 @@ public class TSDBDriverTest { assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); } - public void printRs() { - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet resultSet = statement.executeQuery("show databases"); - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - System.out.print(metaData.getColumnLabel(i) + " : " + resultSet.getString(i) + "\t"); - } - System.out.println(); - } - } catch (SQLException e) { - e.printStackTrace(); - } finally { - try { - if (statement != null) - statement.close(); - if (conn != null) - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - } } \ No newline at end of file From 247f069a0a3841de11aadf3c446537691b481cca Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 08:09:51 +0000 Subject: [PATCH 34/45] rename some functions --- src/dnode/inc/dnodeVRead.h | 8 +- src/dnode/src/dnodeMain.c | 2 +- src/dnode/src/dnodeShell.c | 4 +- src/dnode/src/dnodeVRead.c | 163 +++++++++++++++--------------------- src/dnode/src/dnodeVWrite.c | 40 +++++---- src/inc/dnode.h | 2 +- src/inc/vnode.h | 28 +++---- src/vnode/src/vnodeMain.c | 4 +- src/vnode/src/vnodeRead.c | 14 ++-- src/vnode/src/vnodeWrite.c | 2 +- 10 files changed, 119 insertions(+), 148 deletions(-) diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h index a103520047..b3c3df80b2 100644 --- a/src/dnode/inc/dnodeVRead.h +++ b/src/dnode/inc/dnodeVRead.h @@ -20,9 +20,11 @@ extern "C" { #endif -int32_t dnodeInitVnodeRead(); -void dnodeCleanupVnodeRead(); -void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg); +int32_t dnodeInitVRead(); +void dnodeCleanupVRead(); +void dnodeDispatchToVReadQueue(SRpcMsg *pMsg); +void * dnodeAllocVReadQueue(void *pVnode); +void dnodeFreeVReadQueue(void *rqueue); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index b4c174e29b..5fde4f972b 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -61,7 +61,7 @@ static const SDnodeComponent tsDnodeComponents[] = { {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, {"wal", walInit, walCleanUp}, {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! - {"vread", dnodeInitVnodeRead, dnodeCleanupVnodeRead}, + {"vread", dnodeInitVRead, dnodeCleanupVRead}, {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead}, {"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite}, diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 33cda60793..2f6d844ff1 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -39,8 +39,8 @@ static int32_t tsDnodeSubmitReqNum = 0; int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; // the following message shall be treated as mnode write diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index abf3cb527d..f571cfda9f 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -17,84 +17,79 @@ #include "os.h" #include "taoserror.h" #include "taosmsg.h" -#include "tutil.h" -#include "tqueue.h" -#include "twal.h" #include "tglobal.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" -#include "dnodeVRead.h" +#include "tqueue.h" #include "vnode.h" +#include "dnodeInt.h" typedef struct { - pthread_t thread; // thread - int32_t workerId; // worker ID -} SReadWorker; + pthread_t thread; // thread + int32_t workerId; // worker ID +} SVReadWorker; typedef struct { - int32_t max; // max number of workers - int32_t min; // min number of workers - int32_t num; // current number of workers - SReadWorker *readWorker; + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + SVReadWorker * worker; pthread_mutex_t mutex; -} SReadWorkerPool; +} SVReadWorkerPool; static void *dnodeProcessReadQueue(void *param); -static void dnodeHandleIdleReadWorker(SReadWorker *); // module global variable -static SReadWorkerPool readPool; -static taos_qset readQset; +static SVReadWorkerPool tsVReadWP; +static taos_qset tsVReadQset; -int32_t dnodeInitVnodeRead() { - readQset = taosOpenQset(); +int32_t dnodeInitVRead() { + tsVReadQset = taosOpenQset(); - readPool.min = tsNumOfCores; - readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; - if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; - readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max); - pthread_mutex_init(&readPool.mutex, NULL); + tsVReadWP.min = tsNumOfCores; + tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore; + if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min; + tsVReadWP.worker = (SVReadWorker *)calloc(sizeof(SVReadWorker), tsVReadWP.max); + pthread_mutex_init(&tsVReadWP.mutex, NULL); - if (readPool.readWorker == NULL) return -1; - for (int i = 0; i < readPool.max; ++i) { - SReadWorker *pWorker = readPool.readWorker + i; + if (tsVReadWP.worker == NULL) return -1; + for (int i = 0; i < tsVReadWP.max; ++i) { + SVReadWorker *pWorker = tsVReadWP.worker + i; pWorker->workerId = i; } - dInfo("dnode read is initialized, min worker:%d max worker:%d", readPool.min, readPool.max); + dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max); return 0; } -void dnodeCleanupVnodeRead() { - for (int i = 0; i < readPool.max; ++i) { - SReadWorker *pWorker = readPool.readWorker + i; +void dnodeCleanupVRead() { + for (int i = 0; i < tsVReadWP.max; ++i) { + SVReadWorker *pWorker = tsVReadWP.worker + i; if (pWorker->thread) { - taosQsetThreadResume(readQset); + taosQsetThreadResume(tsVReadQset); } } - for (int i = 0; i < readPool.max; ++i) { - SReadWorker *pWorker = readPool.readWorker + i; + for (int i = 0; i < tsVReadWP.max; ++i) { + SVReadWorker *pWorker = tsVReadWP.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); } } - free(readPool.readWorker); - taosCloseQset(readQset); - pthread_mutex_destroy(&readPool.mutex); + free(tsVReadWP.worker); + taosCloseQset(tsVReadQset); + pthread_mutex_destroy(&tsVReadWP.mutex); - dInfo("dnode read is closed"); + dInfo("dnode vread is closed"); } -void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { - int32_t queuedMsgNum = 0; - int32_t leftLen = pMsg->contLen; - char *pCont = (char *) pMsg->pCont; +void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { + int32_t queuedMsgNum = 0; + int32_t leftLen = pMsg->contLen; + char * pCont = (char *)pMsg->pCont; while (leftLen > 0) { - SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = htonl(pHead->vgId); + SMsgHead *pHead = (SMsgHead *)pCont; + pHead->vgId = htonl(pHead->vgId); pHead->contLen = htonl(pHead->contLen); taos_queue queue = vnodeAcquireRqueue(pHead->vgId); @@ -106,10 +101,10 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { } // put message into queue - SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); - pRead->rpcMsg = *pMsg; - pRead->pCont = pCont; - pRead->contLen = pHead->contLen; + SVReadMsg *pRead = taosAllocateQitem(sizeof(SVReadMsg)); + pRead->rpcMsg = *pMsg; + pRead->pCont = pCont; + pRead->contLen = pHead->contLen; // next vnode leftLen -= pHead->contLen; @@ -120,60 +115,52 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { } if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = { - .handle = pMsg->handle, - .pCont = NULL, - .contLen = 0, - .code = TSDB_CODE_VND_INVALID_VGROUP_ID, - .msgType = 0 - }; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rpcRsp); rpcFreeCont(pMsg->pCont); } } void *dnodeAllocVReadQueue(void *pVnode) { - pthread_mutex_lock(&readPool.mutex); + pthread_mutex_lock(&tsVReadWP.mutex); taos_queue queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&readPool.mutex); + pthread_mutex_unlock(&tsVReadWP.mutex); return NULL; } - taosAddIntoQset(readQset, queue, pVnode); + taosAddIntoQset(tsVReadQset, queue, pVnode); // spawn a thread to process queue - if (readPool.num < readPool.max) { + if (tsVReadWP.num < tsVReadWP.max) { do { - SReadWorker *pWorker = readPool.readWorker + readPool.num; + SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) { - dError("failed to create thread to process read queue, reason:%s", strerror(errno)); + dError("failed to create thread to process vread vqueue since %s", strerror(errno)); } pthread_attr_destroy(&thAttr); - readPool.num++; - dDebug("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num); - } while (readPool.num < readPool.min); + tsVReadWP.num++; + dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num); + } while (tsVReadWP.num < tsVReadWP.min); } - pthread_mutex_unlock(&readPool.mutex); - dDebug("pVnode:%p, read queue:%p is allocated", pVnode, queue); + pthread_mutex_unlock(&tsVReadWP.mutex); + dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue); return queue; } void dnodeFreeVReadQueue(void *rqueue) { taosCloseQueue(rqueue); - - // dynamically adjust the number of threads } -void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { +void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { SRpcMsg rpcRsp = { .handle = pRead->rpcMsg.handle, .pCont = pRead->rspRet.rsp, @@ -186,33 +173,33 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { vnodeRelease(pVnode); } -void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) { +void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } static void *dnodeProcessReadQueue(void *param) { - SReadMsg *pReadMsg; - int type; - void *pVnode; + SVReadMsg *pReadMsg; + int32_t qtype; + void * pVnode; while (1) { - if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) { - dDebug("qset:%p dnode read got no message from qset, exiting", readQset); + if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pReadMsg, &pVnode) == 0) { + dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); break; } dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg); + taosMsg[pReadMsg->rpcMsg.msgType], qtype, pReadMsg); int32_t code = vnodeProcessRead(pVnode, pReadMsg); - if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { - dnodeSendRpcReadRsp(pVnode, pReadMsg, code); + if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { + dnodeSendRpcVReadRsp(pVnode, pReadMsg, code); } else { if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); - } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client + dnodeSendRpcVReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); + } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5)); dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); } @@ -223,19 +210,3 @@ static void *dnodeProcessReadQueue(void *param) { return NULL; } - - -UNUSED_FUNC -static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) { - int32_t num = taosGetQueueNumber(readQset); - - if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) { - readPool.num--; - dDebug("read worker:%d is released, total:%d", pWorker->workerId, readPool.num); - pthread_exit(NULL); - } else { - usleep(30000); - sched_yield(); - } -} - diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 9b58d7bb61..b74dfbfdda 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -15,13 +15,12 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "taoserror.h" +#include "taosmsg.h" #include "tglobal.h" #include "tqueue.h" -#include "tsdb.h" #include "twal.h" -#include "tsync.h" #include "vnode.h" -#include "syncInt.h" #include "dnodeInt.h" typedef struct { @@ -29,22 +28,21 @@ typedef struct { taos_qset qset; // queue set int32_t workerId; // worker ID pthread_t thread; // thread -} SWriteWorker; - +} SVWriteWorker; typedef struct { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic - SWriteWorker *worker; + SVWriteWorker * worker; pthread_mutex_t mutex; -} SWriteWorkerPool; +} SVWriteWorkerPool; -static SWriteWorkerPool tsVWriteWP; -static void *dnodeProcessWriteQueue(void *param); +static SVWriteWorkerPool tsVWriteWP; +static void *dnodeProcessVWriteQueue(void *param); int32_t dnodeInitVWrite() { tsVWriteWP.max = tsNumOfCores; - tsVWriteWP.worker = (SWriteWorker *)tcalloc(sizeof(SWriteWorker), tsVWriteWP.max); + tsVWriteWP.worker = (SVWriteWorker *)tcalloc(sizeof(SVWriteWorker), tsVWriteWP.max); if (tsVWriteWP.worker == NULL) return -1; pthread_mutex_init(&tsVWriteWP.mutex, NULL); @@ -58,14 +56,14 @@ int32_t dnodeInitVWrite() { void dnodeCleanupVWrite() { for (int32_t i = 0; i < tsVWriteWP.max; ++i) { - SWriteWorker *pWorker = tsVWriteWP.worker + i; + SVWriteWorker *pWorker = tsVWriteWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(pWorker->qset); } } for (int32_t i = 0; i < tsVWriteWP.max; ++i) { - SWriteWorker *pWorker = tsVWriteWP.worker + i; + SVWriteWorker *pWorker = tsVWriteWP.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); @@ -100,7 +98,7 @@ void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) { pHead->msgType = pRpcMsg->msgType; pHead->version = 0; pHead->len = pMsg->contLen; - code = vnodeWriteToQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg); + code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg); } if (code != TSDB_CODE_SUCCESS) { @@ -114,7 +112,7 @@ void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) { void *dnodeAllocVWriteQueue(void *pVnode) { pthread_mutex_lock(&tsVWriteWP.mutex); - SWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId; + SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId; void *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&tsVWriteWP.mutex); @@ -141,7 +139,7 @@ void *dnodeAllocVWriteQueue(void *pVnode) { pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessVWriteQueue, pWorker) != 0) { dError("failed to create thread to process vwrite queue since %s", strerror(errno)); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); @@ -190,12 +188,12 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) { vnodeRelease(pVnode); } -static void *dnodeProcessWriteQueue(void *param) { - SWriteWorker *pWorker = param; - SVWriteMsg * pWrite; - void * pVnode; - int32_t numOfMsgs; - int32_t qtype; +static void *dnodeProcessVWriteQueue(void *param) { + SVWriteWorker *pWorker = param; + SVWriteMsg * pWrite; + void * pVnode; + int32_t numOfMsgs; + int32_t qtype; dDebug("dnode vwrite worker:%d is running", pWorker->workerId); diff --git a/src/inc/dnode.h b/src/inc/dnode.h index b4973cc672..6032d8cc0a 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -55,9 +55,9 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid); void *dnodeAllocVWriteQueue(void *pVnode); void dnodeFreeVWriteQueue(void *wqueue); +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); void *dnodeAllocVReadQueue(void *pVnode); void dnodeFreeVReadQueue(void *rqueue); -void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); int32_t dnodeAllocateMnodePqueue(); void dnodeFreeMnodePqueue(); diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 09273d3907..e77bec926a 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -37,20 +37,20 @@ typedef struct { } SRspRet; typedef struct { - SRspRet rspRet; - void *pCont; - int32_t contLen; - SRpcMsg rpcMsg; -} SReadMsg; + SRspRet rspRet; + void * pCont; + int32_t contLen; + SRpcMsg rpcMsg; +} SVReadMsg; typedef struct { - int32_t code; - int32_t processedCount; - void * rpcHandle; - void * rpcAhandle; - SRspRet rspRet; - char reserveForSync[16]; - SWalHead pHead[]; + int32_t code; + int32_t processedCount; + void * rpcHandle; + void * rpcAhandle; + SRspRet rspRet; + char reserveForSync[16]; + SWalHead pHead[]; } SVWriteMsg; extern char *vnodeStatus[]; @@ -66,7 +66,7 @@ void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); -int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); @@ -77,7 +77,7 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); int32_t vnodeInitResources(); void vnodeCleanupResources(); -int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg); +int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pReadMsg); int32_t vnodeCheckRead(void *pVnode); #ifdef __cplusplus diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index a13addb27c..2dcdba5d7c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -266,7 +266,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { strcpy(cqCfg.pass, tsInternalPass); strcpy(cqCfg.db, pVnode->db); cqCfg.vgId = vnode; - cqCfg.cqWrite = vnodeWriteToQueue; + cqCfg.cqWrite = vnodeWriteToWQueue; pVnode->cq = cqOpen(pVnode, &cqCfg); if (pVnode->cq == NULL) { vnodeCleanUp(pVnode); @@ -320,7 +320,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.ahandle = pVnode; syncInfo.getWalInfo = vnodeGetWalInfo; syncInfo.getFileInfo = vnodeGetFileInfo; - syncInfo.writeToCache = vnodeWriteToQueue; + syncInfo.writeToCache = vnodeWriteToWQueue; syncInfo.confirmForward = dnodeSendRpcVWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 99aed03e54..faa35b0e02 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -29,9 +29,9 @@ #include "vnodeInt.h" #include "tqueue.h" -static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg); -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); +static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pReadMsg); +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg); +static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg); static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); void vnodeInitReadFp(void) { @@ -44,7 +44,7 @@ void vnodeInitReadFp(void) { // still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the // request enters the queue // -int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { +int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { SVnodeObj *pVnode = (SVnodeObj *)param; int msgType = pReadMsg->rpcMsg.msgType; @@ -82,7 +82,7 @@ static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) return code; - SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); + SVReadMsg *pRead = (SVReadMsg *)taosAllocateQitem(sizeof(SVReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; pRead->contLen = 0; @@ -146,7 +146,7 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) { pRsp->completed = true; } -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { void * pCont = pReadMsg->pCont; int32_t contLen = pReadMsg->contLen; SRspRet *pRet = &pReadMsg->rspRet; @@ -274,7 +274,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { return code; } -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { +static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { void * pCont = pReadMsg->pCont; SRspRet *pRet = &pReadMsg->rspRet; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 417448423a..80c68b0917 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -205,7 +205,7 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } -int32_t vnodeWriteToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; SWalHead * pHead = wparam; From b037302b3ca9ec2908e56943f1b6b34f478ea5b5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 08:21:22 +0000 Subject: [PATCH 35/45] TD-1949 --- src/wal/src/walWrite.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 470607e965..194c50264f 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -143,12 +143,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { continue; } - if (!pWal->keep) { - wDebug("vgId:%d, file:%s, restore success, remove this file", pWal->vgId, walName); - remove(walName); - } else { - wDebug("vgId:%d, file:%s, restore success and keep it", pWal->vgId, walName); - } + wDebug("vgId:%d, file:%s, restore success and keep it", pWal->vgId, walName); count++; } From f07f456b08e1d5d6308fc2b5ddb23910a9ace3e4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 08:40:53 +0000 Subject: [PATCH 36/45] TD-1948 --- src/dnode/src/dnodeVWrite.c | 4 +++- src/inc/twal.h | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/wal/src/walWrite.c | 7 ++++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index b74dfbfdda..c28ad66b65 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -204,6 +204,7 @@ static void *dnodeProcessVWriteQueue(void *param) { break; } + bool forceFsync = false; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%d version:%" PRIu64, pWrite->rpcAhandle, pWrite, @@ -211,11 +212,12 @@ static void *dnodeProcessVWriteQueue(void *param) { pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); if (pWrite->code <= 0) pWrite->processedCount = 1; + if (pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code)); } - walFsync(vnodeGetWal(pVnode)); + walFsync(vnodeGetWal(pVnode), forceFsync); // browse all items, and process them one by one taosResetQitems(pWorker->qall); diff --git a/src/inc/twal.h b/src/inc/twal.h index d9b5fb26ff..931cf5daba 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -55,7 +55,7 @@ void walStop(twalh); void walClose(twalh); int32_t walRenew(twalh); int32_t walWrite(twalh, SWalHead *); -void walFsync(twalh); +void walFsync(twalh, bool forceFsync); int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); int64_t walGetVersion(twalh); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index fc986521e6..4e370ca028 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -1090,7 +1090,7 @@ static void *sdbWorkerFp(void *param) { } } - walFsync(tsSdbObj.wal); + walFsync(tsSdbObj.wal, true); // browse all items, and process them one by one taosResetQitems(tsSdbWriteQall); diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 194c50264f..b19e084e06 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -111,11 +111,12 @@ int32_t walWrite(void *handle, SWalHead *pHead) { return code; } -void walFsync(void *handle) { +void walFsync(void *handle, bool forceFsync) { SWal *pWal = handle; - if (pWal == NULL || pWal->level != TAOS_WAL_FSYNC || pWal->fd < 0) return; + if (pWal == NULL || pWal->fd < 0) return; - if (pWal->fsyncPeriod == 0) { + if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { + wTrace("vgId:%d, file:%s, do fsync, force:%d", pWal->vgId, pWal->name, forceFsync); if (fsync(pWal->fd) < 0) { wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno)); } From a1fc9d8061378198c1c754e4bfc183ede3439a1e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Nov 2020 16:55:21 +0800 Subject: [PATCH 37/45] [TD-1940] :fix bug in join query. --- src/client/src/tscFunctionImpl.c | 4 ++-- src/query/src/qExecutor.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index e0bd762333..affa4aee83 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3978,7 +3978,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { if (isNull(data1, srcType) || isNull(data2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { - taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); + taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); } } else if (srcType == TSDB_DATA_TYPE_FLOAT) { point1.val = data1; @@ -3987,7 +3987,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { if (isNull(data1, srcType) || isNull(data2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { - taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); + taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); } } else { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 26c4958ca5..b365dc4283 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3843,7 +3843,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); // failed to find data with the specified tag value and vnodeId - if (tsBufIsValidElem(&elem)) { + if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { @@ -4777,7 +4777,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->cur.vgroupIndex == -1) { STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId - if (tsBufIsValidElem(&elem)) { + if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { @@ -4802,7 +4802,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId - if (tsBufIsValidElem(&elem1)) { + if (!tsBufIsValidElem(&elem1)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { From 37f485a98a4273ebd9055c3977bce9bd662dfc71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Nov 2020 17:12:38 +0800 Subject: [PATCH 38/45] [TD-1875]: disable group by in join query. --- src/client/src/tscSQLParser.c | 5 +++++ src/query/tests/tsBufTest.cpp | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f7aaf83066..f497263f2a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2758,6 +2758,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; const char* msg3 = "columns from one table allowed as group by columns"; + const char* msg4 = "join query does not support group by"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2778,6 +2779,10 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (pQueryInfo->numOfTables > 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + STableMeta* pTableMeta = NULL; SSchema* pSchema = NULL; SSchema s = tscGetTbnameColumnSchema(); diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 2892af3979..bada3194bd 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -42,7 +42,7 @@ void simpleTest() { EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->numOfGroups, 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -69,7 +69,7 @@ void largeTSTest() { // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->numOfGroups, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); tsBufFlush(pTSBuf); @@ -105,7 +105,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); - EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->numOfGroups, 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -139,7 +139,7 @@ void multiVnodeTagsTest() { start += step * num; } - EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + EXPECT_EQ(pTSBuf->numOfGroups, j + 1); } EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -184,7 +184,7 @@ void loadDataTest() { start += step * num; } - EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + EXPECT_EQ(pTSBuf->numOfGroups, j + 1); } EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -203,7 +203,7 @@ void loadDataTest() { // create from exists file STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); - EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode); + EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode); EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); @@ -269,7 +269,7 @@ void TSTraverse() { start += step * num; } - EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + EXPECT_EQ(pTSBuf->numOfGroups, j + 1); } tsBufResetPos(pTSBuf); @@ -304,7 +304,7 @@ void TSTraverse() { int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -352,7 +352,7 @@ void TSTraverse() { totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -427,7 +427,7 @@ void mergeDiffVnodeBufferTest() { tsBufFlush(pTSBuf2); tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + EXPECT_EQ(pTSBuf1->numOfGroups, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); tsBufDisplay(pTSBuf1); @@ -472,7 +472,7 @@ void mergeIdenticalVnodeBufferTest() { tsBufFlush(pTSBuf2); tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + EXPECT_EQ(pTSBuf1->numOfGroups, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); tsBufResetPos(pTSBuf1); @@ -482,12 +482,12 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); if (count++ < numOfTags * num) { - EXPECT_EQ(elem.vnode, 12); + EXPECT_EQ(elem.id, 12); } else { - EXPECT_EQ(elem.vnode, 77); + EXPECT_EQ(elem.id, 77); } - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); } tsBufDestroy(pTSBuf1); From fdaf1e470ce98293ceec682b6c251343066c1003 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 17:18:01 +0800 Subject: [PATCH 39/45] [TD-1954]: a inport/export tool for logical backup --- packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ packaging/tools/makeclient.sh | 2 +- packaging/tools/makepkg.sh | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a7bb22f345..edc7de9692 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 54c3c9b279..afec1eaf9a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include @@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 69fa53c087..83a9cb1ced 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a6d868ed1d..00a92cb063 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" From 83f71e2463397bdea2d93e68c3855adf59b08825 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 09:29:26 +0000 Subject: [PATCH 40/45] TD-1847 --- src/wal/src/walMgmt.c | 2 +- src/wal/src/walWrite.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index c8f0274174..8fc7bfeaf3 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -70,7 +70,7 @@ void *walOpen(char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); pthread_mutex_init(&pWal->mutex, NULL); - pWal->fsyncSeq = pCfg->fsyncPeriod % 1000; + pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; if (walInitObj(pWal) != TSDB_CODE_SUCCESS) { diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index b19e084e06..0d27ce1768 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -116,7 +116,7 @@ void walFsync(void *handle, bool forceFsync) { if (pWal == NULL || pWal->fd < 0) return; if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { - wTrace("vgId:%d, file:%s, do fsync, force:%d", pWal->vgId, pWal->name, forceFsync); + wTrace("vgId:%d, file:%s, do fsync", pWal->vgId, pWal->name); if (fsync(pWal->fd) < 0) { wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno)); } From a9593a44727b775bcb9f50a9bde188659a6610d2 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 5 Nov 2020 18:07:26 +0800 Subject: [PATCH 41/45] [TD-1939] add test case for TD-1939 --- tests/pytest/query/queryLike.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/pytest/query/queryLike.py diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py new file mode 100644 index 0000000000..aa943c4f1f --- /dev/null +++ b/tests/pytest/query/queryLike.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))") + tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');") + tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(0) + + tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") + tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") + tdSql.query("select count(*) from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 5ba5336756c6130ee9df1494b648517e96dd90fb Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 5 Nov 2020 18:38:23 +0800 Subject: [PATCH 42/45] update Jenkinksfile --- Jenkinsfile | 1 + tests/pytest/query/queryLike.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8bf7e435fd..834321240f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { cd debug cmake .. > /dev/null make > /dev/null + make install > /dev/null cd ${WKC}/tests #./test-all.sh smoke ./test-all.sh pytest diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py index aa943c4f1f..3c3b030f8f 100644 --- a/tests/pytest/query/queryLike.py +++ b/tests/pytest/query/queryLike.py @@ -33,7 +33,7 @@ class TDTestCase: tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") - tdSql.query("select count(*) from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'") tdSql.checkRows(1) def stop(self): From 6ca2968862967cf55357e604cb7295c8d2809d48 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 10:58:28 +0000 Subject: [PATCH 43/45] TD-1944 --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index d01c34c810..d89d3e275d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1622,7 +1622,7 @@ static void rpcDecRef(SRpcInfo *pRpc) int count = atomic_sub_fetch_32(&tsRpcNum, 1); if (count == 0) { - taosCloseRef(tsRpcRefId); + // taosCloseRef(tsRpcRefId); // tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error } } From 16219bc630431f55e5565b9d6147592d244a4573 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Nov 2020 19:15:20 +0800 Subject: [PATCH 44/45] [TD-225] --- src/client/src/tscSubquery.c | 4 ++++ src/util/src/tcompare.c | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 9905de279e..fd56001174 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -853,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow } if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. + assert(pParentSql->fp != tscJoinQueryCallback); + tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql); freeJoinSubqueryObj(pParentSql); // set no result command pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + assert(pParentSql->fp != tscJoinQueryCallback); + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { // proceed to for ts_comp query diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index ba711ced8f..1090ab8101 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -367,7 +367,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); case TSDB_DATA_TYPE_NCHAR: { - int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE); + tstr* t1 = (tstr*) f1; + tstr* t2 = (tstr*) f2; + + if (t1->len != t2->len) { + return t1->len > t2->len? 1:-1; + } + + int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE); if (ret == 0) { return ret; } From ea374b6f868ef2cfe6efa51238bafadb03042359 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 11:34:40 +0000 Subject: [PATCH 45/45] TD-1915 --- src/dnode/src/dnodeVRead.c | 48 +++++++------------ src/inc/vnode.h | 17 ++++--- src/vnode/src/vnodeMain.c | 15 ------ src/vnode/src/vnodeRead.c | 97 +++++++++++++++++++++++++------------- src/vnode/src/vnodeWrite.c | 2 +- 5 files changed, 94 insertions(+), 85 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index f571cfda9f..34df11adcc 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -92,33 +92,23 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { pHead->vgId = htonl(pHead->vgId); pHead->contLen = htonl(pHead->contLen); - taos_queue queue = vnodeAcquireRqueue(pHead->vgId); - - if (queue == NULL) { - leftLen -= pHead->contLen; - pCont -= pHead->contLen; - continue; + void *pVnode = vnodeAcquire(pHead->vgId); + if (pVnode != NULL) { + int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; + vnodeRelease(pVnode); } - // put message into queue - SVReadMsg *pRead = taosAllocateQitem(sizeof(SVReadMsg)); - pRead->rpcMsg = *pMsg; - pRead->pCont = pCont; - pRead->contLen = pHead->contLen; - - // next vnode leftLen -= pHead->contLen; pCont -= pHead->contLen; - queuedMsgNum++; - - taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); } if (queuedMsgNum == 0) { SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); } void *dnodeAllocVReadQueue(void *pVnode) { @@ -162,50 +152,48 @@ void dnodeFreeVReadQueue(void *rqueue) { void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { SRpcMsg rpcRsp = { - .handle = pRead->rpcMsg.handle, + .handle = pRead->rpcHandle, .pCont = pRead->rspRet.rsp, .contLen = pRead->rspRet.len, .code = code, }; rpcSendResponse(&rpcRsp); - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } static void *dnodeProcessReadQueue(void *param) { - SVReadMsg *pReadMsg; + SVReadMsg *pRead; int32_t qtype; void * pVnode; while (1) { - if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pReadMsg, &pVnode) == 0) { + if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) { dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); break; } - dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], qtype, pReadMsg); + dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead, + taosMsg[pRead->msgType], qtype); - int32_t code = vnodeProcessRead(pVnode, pReadMsg); + int32_t code = vnodeProcessRead(pVnode, pRead); if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { - dnodeSendRpcVReadRsp(pVnode, pReadMsg, code); + dnodeSendRpcVReadRsp(pVnode, pRead, code); } else { if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcVReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); + dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code); } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5)); - dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5)); + dnodeDispatchNonRspMsg(pVnode, pRead, code); } } - taosFreeQitem(pReadMsg); + taosFreeQitem(pRead); } return NULL; diff --git a/src/inc/vnode.h b/src/inc/vnode.h index e77bec926a..018e96e193 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -37,10 +37,15 @@ typedef struct { } SRspRet; typedef struct { - SRspRet rspRet; - void * pCont; + int32_t code; int32_t contLen; - SRpcMsg rpcMsg; + void * rpcHandle; + void * rpcAhandle; + void * qhandle; + int8_t qtype; + int8_t msgType; + SRspRet rspRet; + char pCont[]; } SVReadMsg; typedef struct { @@ -62,13 +67,11 @@ int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); void* vnodeAcquire(int32_t vgId); // add refcount -void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam); -int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); void vnodeConfirmForward(void *param, uint64_t version, int32_t code); @@ -77,8 +80,8 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); int32_t vnodeInitResources(); void vnodeCleanupResources(); -int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pReadMsg); -int32_t vnodeCheckRead(void *pVnode); +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam); +int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); #ifdef __cplusplus } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 2dcdba5d7c..128da72623 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -468,21 +468,6 @@ void *vnodeAcquire(int32_t vgId) { return *ppVnode; } -void *vnodeAcquireRqueue(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) return NULL; - - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]); - vnodeRelease(pVnode); - return NULL; - } - - return pVnode->rqueue; -} - void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index faa35b0e02..fb984f6750 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -46,7 +46,7 @@ void vnodeInitReadFp(void) { // int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { SVnodeObj *pVnode = (SVnodeObj *)param; - int msgType = pReadMsg->rpcMsg.msgType; + int32_t msgType = pReadMsg->msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); @@ -56,7 +56,7 @@ int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -int32_t vnodeCheckRead(void *param) { +static int32_t vnodeCheckRead(void *param) { SVnodeObj *pVnode = param; if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], @@ -78,24 +78,58 @@ int32_t vnodeCheckRead(void *param) { return TSDB_CODE_SUCCESS; } -static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - SVReadMsg *pRead = (SVReadMsg *)taosAllocateQitem(sizeof(SVReadMsg)); - pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; - pRead->pCont = qhandle; - pRead->contLen = 0; - pRead->rpcMsg.ahandle = ahandle; +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + + if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) { + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; + } + + int32_t size = sizeof(SVReadMsg) + contLen; + SVReadMsg *pRead = taosAllocateQitem(size); + if (pRead == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + if (rparam != NULL) { + SRpcMsg *pRpcMsg = rparam; + pRead->rpcHandle = pRpcMsg->handle; + pRead->rpcAhandle = pRpcMsg->ahandle; + pRead->msgType = pRpcMsg->msgType; + pRead->code = pRpcMsg->code; + } + + if (contLen != 0) { + pRead->contLen = contLen; + memcpy(pRead->pCont, pCont, contLen); + } else { + pRead->qhandle = pCont; + } + + pRead->qtype = qtype; atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); - taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); - + taosWriteQitem(pVnode->rqueue, qtype, pRead); return TSDB_CODE_SUCCESS; } +static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + rpcMsg.ahandle = ahandle; + + int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); + if (code == TSDB_CODE_SUCCESS) { + vDebug("QInfo:%p add to vread queue for exec query", *qhandle); + } + + return code; +} + /** * * @param pRet response message object @@ -155,18 +189,18 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { memset(pRet, 0, sizeof(SRspRet)); // qHandle needs to be freed correctly - if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont; killQueryMsg->free = htons(killQueryMsg->free); killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); - vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); - assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); + vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle); + assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1); void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle); if (qhandle == NULL || *qhandle == NULL) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); } else { assert(*qhandle == (void *)killQueryMsg->qhandle); @@ -208,9 +242,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } if (handle != NULL && - vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return pRsp->code; @@ -221,7 +255,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle); + code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -230,7 +264,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } } else { assert(pCont != NULL); - void **qhandle = (void **)pCont; + void **qhandle = (void **)pReadMsg->qhandle; vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); @@ -242,14 +276,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); - assert(pReadMsg->rpcMsg.handle != NULL); + pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pReadMsg->rpcHandle != NULL); vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); // set the real rsp error code - pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle); + pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -283,7 +317,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, - pRetrieve->free, pReadMsg->rpcMsg.handle); + pRetrieve->free, pReadMsg->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); @@ -314,9 +348,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } // register the qhandle to connect to quit query immediate if connection is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -326,7 +359,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { bool freeHandle = true; bool buildRes = false; - code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle); + code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle); if (code != TSDB_CODE_SUCCESS) { // TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -337,7 +370,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { assert(buildRes == true); #if _NON_BLOCKING_RETRIEVE if (!buildRes) { - assert(pReadMsg->rpcMsg.handle != NULL); + assert(pReadMsg->rpcHandle != NULL); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; @@ -345,7 +378,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { #endif // ahandle is the sqlObj pointer - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 80c68b0917..3caee2fb0c 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -97,7 +97,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara return syncCode; } -int32_t vnodeCheckWrite(void *param) { +static int32_t vnodeCheckWrite(void *param) { SVnodeObj *pVnode = param; if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);