From 920ff4d40cfabaed0cd0b6a072548eabb77da8a0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 May 2021 11:37:49 +0800 Subject: [PATCH 01/25] [td-4175]: improve the error message while the binary string overflow. --- src/client/src/tscParseInsert.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 923d95c888..f09ca351ff 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -709,6 +709,9 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock int32_t numOfRows = 0; code = tsParseValues(str, dataBuf, maxNumOfRows, pCmd, &numOfRows, tmpTokenBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { SParamInfo *param = dataBuf->params + i; From 2b70b54cd4749d198b4f6f01db91325a5303695a Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 13 May 2021 15:27:33 +0800 Subject: [PATCH 02/25] fix bug --- src/client/src/tscProfile.c | 4 ++-- src/client/src/tscUtil.c | 8 ++++---- src/query/src/qExecutor.c | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index be304632cc..ea2a3d8228 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -54,14 +54,14 @@ void tscAddIntoSqlList(SSqlObj *pSql) { pSql->next = pObj->sqlList; if (pObj->sqlList) pObj->sqlList->prev = pSql; pObj->sqlList = pSql; - pSql->queryId = queryId++; + pSql->queryId = atomic_fetch_add_32(&queryId, 1); pthread_mutex_unlock(&pObj->mutex); pSql->stime = taosGetTimestampMs(); pSql->listed = 1; - tscDebug("0x%"PRIx64" added into sqlList", pSql->self); + tscDebug("0x%"PRIx64" added into sqlList, queryId:%u", pSql->self, pSql->queryId); } void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 547d66a0f3..3d2cf44560 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2431,15 +2431,15 @@ void tscDoQuery(SSqlObj* pSql) { return; } - if (pCmd->command == TSDB_SQL_SELECT) { - tscAddIntoSqlList(pSql); - } - if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { tscImportDataFromFile(pSql); } else { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; + + if ((pCmd->command == TSDB_SQL_SELECT) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_STABLE_SUBQUERY))) { + tscAddIntoSqlList(pSql); + } if (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_INSERT)) { // multi-vnodes insertion tscHandleMultivnodeInsert(pSql); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4cf6d6072e..eca2a25a35 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -124,6 +124,8 @@ int64_t genQueryId(void) { uid |= sid; + qDebug("gen qid:0x%"PRIx64, uid); + return uid; } From 57aceae597c513c72736dea7efc75d37c315092c Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 13 May 2021 16:48:37 +0800 Subject: [PATCH 03/25] fix bug --- src/mnode/src/mnodeProfile.c | 5 +++-- src/mnode/src/mnodeShow.c | 4 ---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 17a4282d05..f0ff016694 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -123,8 +123,9 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { mDebug("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user, taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port); - taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); - return NULL; + pConn->port = port; + //taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); + //return NULL; } // mDebug("connId:%d, is incoming, user:%s ip:%s:%u", connId, pConn->user, taosIpStr(pConn->ip), pConn->port); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index c0fa6368f3..5fe22826b7 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -253,10 +253,6 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { int32_t connId = htonl(pHBMsg->connId); SConnObj *pConn = mnodeAccquireConn(connId, connInfo.user, connInfo.clientIp, connInfo.clientPort); - if (pConn == NULL) { - pHBMsg->pid = htonl(pHBMsg->pid); - pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pHBMsg->pid, pHBMsg->appName); - } if (pConn == NULL) { // do not close existing links, otherwise From ee36db4af9a5cf3fe4f81ca535403a808ae7b50c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 13 May 2021 10:09:04 +0000 Subject: [PATCH 04/25] [TD-4167]add smoke test on centos7 in CI --- .drone.yml | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/.drone.yml b/.drone.yml index e7ae6ebbda..d13aad3153 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,41 +7,22 @@ platform: arch: amd64 steps: -- name: smoke_test - image: python:3.8 +- name: build + image: gcc commands: - apt-get update - - apt-get install -y cmake build-essential gcc - - pip3 install psutil - - pip3 install guppy3 - - pip3 install src/connector/python/linux/python3/ + - apt-get install -y cmake build-essential - mkdir debug - cd debug - cmake .. - make - - cd ../tests - - ./test-all.sh smoke + trigger: + event: + - pull_request when: branch: - develop - master - - -- name: crash_gen - image: python:3.8 - commands: - - pip3 install requests - - pip3 install src/connector/python/linux/python3/ - - pip3 install psutil - - pip3 install guppy3 - - cd tests/pytest - - ./crash_gen.sh -a -p -t 4 -s 2000 - when: - branch: - - develop - - master - - --- kind: pipeline name: test_arm64 @@ -60,6 +41,9 @@ steps: - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - make + trigger: + event: + - pull_request when: branch: - develop @@ -82,6 +66,9 @@ steps: - cd debug - cmake .. -DCPUTYPE=aarch32 > /dev/null - make + trigger: + event: + - pull_request when: branch: - develop @@ -106,11 +93,13 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop - master - --- kind: pipeline name: build_xenial @@ -129,6 +118,9 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop @@ -151,6 +143,9 @@ steps: - cd debug - cmake .. - make + trigger: + event: + - pull_request when: branch: - develop From 770bfd7b39c3c0bb105dffb291858792b3c1cd68 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 May 2021 22:55:12 +0800 Subject: [PATCH 05/25] [TD-4136]: taosdemo max records per req < 32767 (#6118) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 49d89dc597..72044753d8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4661,7 +4661,7 @@ static int64_t generateData(char *recBuf, char **data_type, double t = rand_double(); pstr += sprintf(pstr, ",%20.8f", t); } else if (strcasecmp(data_type[i % c], "BOOL") == 0) { - bool b = taosRandom() & 1; + bool b = rand_bool() & 1; pstr += sprintf(pstr, ",%s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "BINARY") == 0) { char *s = malloc(lenOfBinary); From e08fe44fdfebe0cfef0e52b062d88734ed6acbc0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 09:38:04 +0800 Subject: [PATCH 06/25] DB/create: fix create database when default quorum > replica --- src/mnode/src/mnodeDb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..7cc5d41ae1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; From db6c390289bdf2196ef2243896b3fe15f7cd25f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 May 2021 09:52:07 +0800 Subject: [PATCH 07/25] TD-4176 --- src/wal/src/walWrite.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 4368ddd7d3..ca19a39f2b 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -426,8 +426,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch #endif offset = offset + sizeof(SWalHead) + pHead->len; - wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, - fileId, pHead->version, pWal->version, pHead->len); + wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64, + pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset); pWal->version = pHead->version; (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); From 5d4b05ecd1dbf46254e77d3b4776db45925f8e2a Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Fri, 14 May 2021 10:00:05 +0800 Subject: [PATCH 08/25] Fix/td 4148 (#6122) * [TD-4148]: JDBC-Restful Compatible version before 2.0.18.0 * [TD-4174]: add test case for TD-4174 * change * change * change * change * change --- .../java/com/taosdata/jdbc/TSDBConstants.java | 1 + .../taosdata/jdbc/rs/RestfulResultSet.java | 152 ++++++++++++++---- .../taosdata/jdbc/utils/UtcTimestampUtil.java | 12 -- .../java/com/taosdata/jdbc/utils/Utils.java | 52 ++++++ .../InsertSpecialCharacterRestfulTest.java | 1 - .../com/taosdata/jdbc/cases/TD4174Test.java | 62 +++++++ ...woTypeTimestampPercisionInRestfulTest.java | 6 +- .../jdbc/cases/UnsignedNumberJniTest.java | 92 +++++------ .../jdbc/cases/UnsignedNumberRestfulTest.java | 105 ++++++------ .../jdbc/rs/RestfulPreparedStatementTest.java | 2 - .../jdbc/rs/RestfulResultSetTest.java | 6 +- 11 files changed, 335 insertions(+), 156 deletions(-) delete mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 37073e243f..f38555ce8a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -30,6 +30,7 @@ public abstract class TSDBConstants { public static final int JNI_FETCH_END = -6; public static final int JNI_OUT_OF_MEMORY = -7; // TSDB Data Types + public static final int TSDB_DATA_TYPE_NULL = 0; public static final int TSDB_DATA_TYPE_BOOL = 1; public static final int TSDB_DATA_TYPE_TINYINT = 2; public static final int TSDB_DATA_TYPE_SMALLINT = 3; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index db635f5f79..530b433d42 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -6,11 +6,13 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; import com.taosdata.jdbc.*; +import com.taosdata.jdbc.utils.Utils; import java.math.BigDecimal; import java.sql.*; import java.time.Instant; import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Calendar; @@ -18,14 +20,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { private volatile boolean isClosed; private int pos = -1; - private final String database; private final Statement statement; // data - private final ArrayList> resultSet; + private final ArrayList> resultSet = new ArrayList<>(); // meta - private ArrayList columnNames; - private ArrayList columns; + private ArrayList columnNames = new ArrayList<>(); + private ArrayList columns = new ArrayList<>(); private RestfulResultSetMetaData metaData; /** @@ -37,10 +38,46 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { this.database = database; this.statement = statement; - // column metadata + // get column metadata JSONArray columnMeta = resultJson.getJSONArray("column_meta"); - columnNames = new ArrayList<>(); - columns = new ArrayList<>(); + // get row data + JSONArray data = resultJson.getJSONArray("data"); + if (data == null || data.isEmpty()) { + columnNames.clear(); + columns.clear(); + this.resultSet.clear(); + return; + } + // get head + JSONArray head = resultJson.getJSONArray("head"); + // get rows + Integer rows = resultJson.getInteger("rows"); + // parse column_meta + if (columnMeta != null) { + parseColumnMeta_new(columnMeta); + } else { + parseColumnMeta_old(head, data, rows); + } + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + // parse row data + resultSet.clear(); + for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { + ArrayList row = new ArrayList(); + JSONArray jsonRow = data.getJSONArray(rowIndex); + for (int colIndex = 0; colIndex < this.metaData.getColumnCount(); colIndex++) { + row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).taos_type)); + } + resultSet.add(row); + } + } + + /*** + * use this method after TDengine-2.0.18.0 to parse column meta, restful add column_meta in resultSet + * @Param columnMeta + */ + private void parseColumnMeta_new(JSONArray columnMeta) throws SQLException { + columnNames.clear(); + columns.clear(); for (int colIndex = 0; colIndex < columnMeta.size(); colIndex++) { JSONArray col = columnMeta.getJSONArray(colIndex); String col_name = col.getString(0); @@ -50,23 +87,55 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { columnNames.add(col_name); columns.add(new Field(col_name, col_type, col_length, "", taos_type)); } - this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + } - // row data - JSONArray data = resultJson.getJSONArray("data"); - resultSet = new ArrayList<>(); - for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { - ArrayList row = new ArrayList(); - JSONArray jsonRow = data.getJSONArray(rowIndex); - for (int colIndex = 0; colIndex < jsonRow.size(); colIndex++) { - row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).taos_type)); + /** + * use this method before TDengine-2.0.18.0 to parse column meta + */ + private void parseColumnMeta_old(JSONArray head, JSONArray data, int rows) { + columnNames.clear(); + columns.clear(); + for (int colIndex = 0; colIndex < head.size(); colIndex++) { + String col_name = head.getString(colIndex); + columnNames.add(col_name); + + int col_type = Types.NULL; + int col_length = 0; + int taos_type = TSDBConstants.TSDB_DATA_TYPE_NULL; + + JSONArray row0Json = data.getJSONArray(0); + if (colIndex < row0Json.size()) { + Object value = row0Json.get(colIndex); + if (value instanceof Boolean) { + col_type = Types.BOOLEAN; + col_length = 1; + taos_type = TSDBConstants.TSDB_DATA_TYPE_BOOL; + } + if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) { + col_type = Types.BIGINT; + col_length = 8; + taos_type = TSDBConstants.TSDB_DATA_TYPE_BIGINT; + } + if (value instanceof Float || value instanceof Double || value instanceof BigDecimal) { + col_type = Types.DOUBLE; + col_length = 8; + taos_type = TSDBConstants.TSDB_DATA_TYPE_DOUBLE; + } + if (value instanceof String) { + col_type = Types.NCHAR; + col_length = ((String) value).length(); + taos_type = TSDBConstants.TSDB_DATA_TYPE_NCHAR; + } } - resultSet.add(row); + columns.add(new Field(col_name, col_type, col_length, "", taos_type)); } } + private Object parseColumnData(JSONArray row, int colIndex, int taosType) throws SQLException { switch (taosType) { + case TSDBConstants.TSDB_DATA_TYPE_NULL: + return null; case TSDBConstants.TSDB_DATA_TYPE_BOOL: return row.getBoolean(colIndex); case TSDBConstants.TSDB_DATA_TYPE_TINYINT: @@ -290,8 +359,10 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return 0; } wasNull = false; - if (value instanceof Float || value instanceof Double) + if (value instanceof Float) return (float) value; + if (value instanceof Double) + return new Float((Double) value); return Float.parseFloat(value.toString()); } @@ -329,6 +400,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return Shorts.toByteArray((short) value); if (value instanceof Byte) return new byte[]{(byte) value}; + if (value instanceof Timestamp) { + return Utils.formatTimestamp((Timestamp) value).getBytes(); + } return value.toString().getBytes(); } @@ -342,7 +416,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return new Date(((Timestamp) value).getTime()); - return Date.valueOf(value.toString()); + Date date = null; + date = Utils.parseDate(value.toString()); + return date; } @Override @@ -354,7 +430,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return new Time(((Timestamp) value).getTime()); - return Time.valueOf(value.toString()); + Time time = null; + try { + time = Utils.parseTime(value.toString()); + } catch (DateTimeParseException e) { + time = null; + } + return time; } @Override @@ -366,14 +448,20 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return (Timestamp) value; -// if (value instanceof Long) { -// if (1_0000_0000_0000_0L > (long) value) -// return Timestamp.from(Instant.ofEpochMilli((long) value)); -// long epochSec = (long) value / 1000_000L; -// long nanoAdjustment = (long) ((long) value % 1000_000L * 1000); -// return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); -// } - return Timestamp.valueOf(value.toString()); + if (value instanceof Long) { + if (1_0000_0000_0000_0L > (long) value) + return Timestamp.from(Instant.ofEpochMilli((long) value)); + long epochSec = (long) value / 1000_000L; + long nanoAdjustment = (long) value % 1000_000L * 1000; + return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + } + Timestamp ret; + try { + ret = Utils.parseTimestamp(value.toString()); + } catch (Exception e) { + ret = null; + } + return ret; } @Override @@ -415,7 +503,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return new BigDecimal(Double.valueOf(value.toString())); if (value instanceof Timestamp) return new BigDecimal(((Timestamp) value).getTime()); - return new BigDecimal(value.toString()); + BigDecimal ret; + try { + ret = new BigDecimal(value.toString()); + } catch (Exception e) { + ret = null; + } + return ret; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java deleted file mode 100644 index 04a11a2beb..0000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; - -public class UtcTimestampUtil { - public static final DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-ddTHH:mm:ss.SSS+") -// .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) - .toFormatter(); - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index 0ce4bd3dc1..082cec1e24 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -5,7 +5,15 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import java.nio.charset.Charset; +import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -17,6 +25,41 @@ public class Utils { private static Pattern ptn = Pattern.compile(".*?'"); + private static final DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss.SSS").toFormatter(); + private static final DateTimeFormatter formatter2 = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss.SSSSSS").toFormatter(); + + public static Time parseTime(String timestampStr) throws DateTimeParseException { + LocalTime time; + try { + time = LocalTime.parse(timestampStr, formatter); + } catch (DateTimeParseException e) { + time = LocalTime.parse(timestampStr, formatter2); + } + return Time.valueOf(time); + } + + public static Date parseDate(String timestampStr) throws DateTimeParseException { + LocalDate date; + try { + date = LocalDate.parse(timestampStr, formatter); + } catch (DateTimeParseException e) { + date = LocalDate.parse(timestampStr, formatter2); + } + return Date.valueOf(date); + } + + public static Timestamp parseTimestamp(String timeStampStr) { + LocalDateTime dateTime; + try { + dateTime = LocalDateTime.parse(timeStampStr, formatter); + } catch (DateTimeParseException e) { + dateTime = LocalDateTime.parse(timeStampStr, formatter2); + } + return Timestamp.valueOf(dateTime); + } + public static String escapeSingleQuota(String origin) { Matcher m = ptn.matcher(origin); StringBuffer sb = new StringBuffer(); @@ -132,4 +175,13 @@ public class Utils { }).collect(Collectors.joining()); } + + public static String formatTimestamp(Timestamp timestamp) { + int nanos = timestamp.getNanos(); + if (nanos % 1000000l != 0) + return timestamp.toLocalDateTime().format(formatter2); + return timestamp.toLocalDateTime().format(formatter); + } + + } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java index ea0d1aec41..fa6cbd22b5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java @@ -7,7 +7,6 @@ import java.sql.*; public class InsertSpecialCharacterRestfulTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; private static String dbName = "spec_char_test"; private static String tbname1 = "test"; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java new file mode 100644 index 0000000000..2704d4cfa5 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java @@ -0,0 +1,62 @@ +package com.taosdata.jdbc.cases; + +import com.alibaba.fastjson.JSONObject; +import com.taosdata.jdbc.TSDBDriver; +import org.junit.*; + +import java.sql.*; +import java.util.Properties; + +public class TD4174Test { + private Connection conn; + private static final String host = "127.0.0.1"; + + @Test + public void test() { + long ts = System.currentTimeMillis(); + try (PreparedStatement pstmt = conn.prepareStatement("insert into weather values(" + ts + ", ?)")) { + JSONObject value = new JSONObject(); + value.put("name", "John Smith"); + value.put("age", 20); + Assert.assertEquals("{\"name\":\"John Smith\",\"age\":20}",value.toJSONString()); + pstmt.setString(1, value.toJSONString()); + + int ret = pstmt.executeUpdate(); + Assert.assertEquals(1, ret); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + JSONObject value = new JSONObject(); + value.put("name", "John Smith"); + value.put("age", 20); + System.out.println(value.toJSONString()); + } + + @Before + public void before() throws SQLException { + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + conn = DriverManager.getConnection(url, properties); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists td4174"); + stmt.execute("create database if not exists td4174"); + stmt.execute("use td4174"); + stmt.execute("create table weather(ts timestamp, text binary(64))"); + } + } + + @After + public void after() throws SQLException { + if (conn != null) + conn.close(); + + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java index ed4f979ef3..5c83b5a9da 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java @@ -13,6 +13,7 @@ import java.util.Properties; public class TwoTypeTimestampPercisionInRestfulTest { private static final String host = "127.0.0.1"; + private static final String ms_timestamp_db = "ms_precision_test"; private static final String us_timestamp_db = "us_precision_test"; private static final long timestamp1 = System.currentTimeMillis(); @@ -94,7 +95,8 @@ public class TwoTypeTimestampPercisionInRestfulTest { try (Statement stmt = conn3.createStatement()) { ResultSet rs = stmt.executeQuery("select last_row(ts) from " + ms_timestamp_db + ".weather"); rs.next(); - long ts = rs.getTimestamp(1).getTime(); + Timestamp actual = rs.getTimestamp(1); + long ts = actual == null ? 0 : actual.getTime(); Assert.assertEquals(timestamp1, ts); ts = rs.getLong(1); Assert.assertEquals(timestamp1, ts); @@ -110,7 +112,7 @@ public class TwoTypeTimestampPercisionInRestfulTest { rs.next(); Timestamp timestamp = rs.getTimestamp(1); - long ts = timestamp.getTime(); + long ts = timestamp == null ? 0 : timestamp.getTime(); Assert.assertEquals(timestamp1, ts); int nanos = timestamp.getNanos(); Assert.assertEquals(timestamp2 % 1000_000l * 1000, nanos); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java index d1816a3e7c..fb23c0e64a 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java @@ -9,19 +9,19 @@ import java.util.Properties; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class UnsignedNumberJniTest { + private static final String host = "127.0.0.1"; private static Connection conn; + private static long ts; @Test public void testCase001() { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); Assert.assertEquals("127", rs.getString(2)); Assert.assertEquals("32767", rs.getString(3)); Assert.assertEquals("2147483647", rs.getString(4)); @@ -37,13 +37,10 @@ public class UnsignedNumberJniTest { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); Assert.assertEquals(2147483647, rs.getInt(4)); @@ -61,16 +58,14 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,2147483647, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getLong(5); } } } @@ -82,15 +77,15 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getInt(4); } } } @@ -102,15 +97,15 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + assertResultSetMetaData(meta); + while (rs.next()) { + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals("65534", rs.getString(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getShort(3); } } } @@ -122,37 +117,27 @@ public class UnsignedNumberJniTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); - } - } - } + assertResultSetMetaData(meta); - @Test - public void testCase007() throws SQLException { - try (Statement stmt = conn.createStatement()) { - long now = System.currentTimeMillis(); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); - ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); - ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals("254", rs.getString(2)); Assert.assertEquals("65534", rs.getString(3)); Assert.assertEquals("4294967294", rs.getString(4)); Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getByte(2); } } } + private void assertResultSetMetaData(ResultSetMetaData meta) throws SQLException { + Assert.assertEquals(5, meta.getColumnCount()); + Assert.assertEquals("ts", meta.getColumnLabel(1)); + Assert.assertEquals("f1", meta.getColumnLabel(2)); + Assert.assertEquals("f2", meta.getColumnLabel(3)); + Assert.assertEquals("f3", meta.getColumnLabel(4)); + Assert.assertEquals("f4", meta.getColumnLabel(5)); + } @BeforeClass public static void beforeClass() { @@ -160,20 +145,19 @@ public class UnsignedNumberJniTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + ts = System.currentTimeMillis(); try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; conn = DriverManager.getConnection(url, properties); - Statement stmt = conn.createStatement(); stmt.execute("drop database if exists unsign_jni"); stmt.execute("create database if not exists unsign_jni"); stmt.execute("use unsign_jni"); stmt.execute("create table us_table(ts timestamp, f1 tinyint unsigned, f2 smallint unsigned, f3 int unsigned, f4 bigint unsigned)"); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(now, 127, 32767,2147483647, 9223372036854775807)"); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + ts + ", 127, 32767,2147483647, 9223372036854775807)"); stmt.close(); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { e.printStackTrace(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java index 4ae2f36fe9..a659a490cb 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java @@ -13,17 +13,20 @@ public class UnsignedNumberRestfulTest { private static final String host = "127.0.0.1"; private static Connection conn; + private static long ts; @Test public void testCase001() { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); + Assert.assertEquals("127", rs.getString(2)); + Assert.assertEquals("32767", rs.getString(3)); + Assert.assertEquals("2147483647", rs.getString(4)); + Assert.assertEquals("9223372036854775807", rs.getString(5)); } } catch (SQLException e) { e.printStackTrace(); @@ -35,13 +38,14 @@ public class UnsignedNumberRestfulTest { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select * from us_table"); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals(9223372036854775807l, rs.getLong(5)); } } catch (SQLException e) { e.printStackTrace(); @@ -55,13 +59,14 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,2147483647, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getLong(5); } } } @@ -73,13 +78,15 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getInt(4); } } } @@ -91,13 +98,15 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); + assertResultSetMetaData(meta); + while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals("65534", rs.getString(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getShort(3); } } } @@ -109,57 +118,47 @@ public class UnsignedNumberRestfulTest { stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); - System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); - System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); - System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); - System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); - System.out.println(); - } - } - } + assertResultSetMetaData(meta); - @Test - public void testCase007() throws SQLException { - try (Statement stmt = conn.createStatement()) { - long now = System.currentTimeMillis(); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); - ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); - ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals(now, rs.getTimestamp(1).getTime()); Assert.assertEquals("254", rs.getString(2)); Assert.assertEquals("65534", rs.getString(3)); Assert.assertEquals("4294967294", rs.getString(4)); Assert.assertEquals("18446744073709551614", rs.getString(5)); + rs.getByte(2); } } } + private void assertResultSetMetaData(ResultSetMetaData meta) throws SQLException { + Assert.assertEquals(5, meta.getColumnCount()); + Assert.assertEquals("ts", meta.getColumnLabel(1)); + Assert.assertEquals("f1", meta.getColumnLabel(2)); + Assert.assertEquals("f2", meta.getColumnLabel(3)); + Assert.assertEquals("f3", meta.getColumnLabel(4)); + Assert.assertEquals("f4", meta.getColumnLabel(5)); + } + @BeforeClass public static void beforeClass() { Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + ts = System.currentTimeMillis(); try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; conn = DriverManager.getConnection(url, properties); - Statement stmt = conn.createStatement(); stmt.execute("drop database if exists unsign_restful"); stmt.execute("create database if not exists unsign_restful"); stmt.execute("use unsign_restful"); stmt.execute("create table us_table(ts timestamp, f1 tinyint unsigned, f2 smallint unsigned, f3 int unsigned, f4 bigint unsigned)"); - stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(now, 127, 32767,2147483647, 9223372036854775807)"); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + ts + ", 127, 32767,2147483647, 9223372036854775807)"); stmt.close(); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { e.printStackTrace(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java index e4dd6384f9..ee457ff412 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -10,7 +10,6 @@ import java.sql.*; public class RestfulPreparedStatementTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static PreparedStatement pstmt_insert; @@ -371,7 +370,6 @@ public class RestfulPreparedStatementTest { pstmt_insert.setSQLXML(1, null); } - @BeforeClass public static void beforeClass() { try { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index 9bfe9a04ff..81e762c5ca 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -18,7 +18,6 @@ import java.text.SimpleDateFormat; public class RestfulResultSetTest { private static final String host = "127.0.0.1"; - private static Connection conn; private static Statement stmt; private static ResultSet rs; @@ -95,7 +94,8 @@ public class RestfulResultSetTest { @Test public void getBigDecimal() throws SQLException { BigDecimal f1 = rs.getBigDecimal("f1"); - Assert.assertEquals(1609430400000l, f1.longValue()); + long actual = (f1 == null) ? 0 : f1.longValue(); + Assert.assertEquals(1609430400000l, actual); BigDecimal f2 = rs.getBigDecimal("f2"); Assert.assertEquals(1, f2.intValue()); @@ -119,7 +119,7 @@ public class RestfulResultSetTest { @Test public void getBytes() throws SQLException { byte[] f1 = rs.getBytes("f1"); - Assert.assertEquals("2021-01-01 00:00:00.0", new String(f1)); + Assert.assertEquals("2021-01-01 00:00:00.000", new String(f1)); byte[] f2 = rs.getBytes("f2"); Assert.assertEquals(1, Ints.fromByteArray(f2)); From 4681a7bf7c87d64ebd136c86747aff5072e214b0 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 14 May 2021 02:00:12 +0000 Subject: [PATCH 09/25] add build test in centos --- .drone.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.drone.yml b/.drone.yml index d13aad3153..dd90a0440e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -150,6 +150,29 @@ steps: branch: - develop - master +--- +kind: pipeline +name: build_centos7 +platform: + os: linux + arch: amd64 + +steps: +- name: build + image: ansible/centos7-ansible + commands: + - yum install gcc cmake build-essential -y + - mkdir debug + - cd debug + - cmake .. + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master --- kind: pipeline From 8c06d5952a70589804562409eef0e2a3171545b5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 10:07:35 +0800 Subject: [PATCH 10/25] [TD-4081]: fix vnode dropping --- src/vnode/src/vnodeMain.c | 2 -- src/vnode/src/vnodeMgmt.c | 2 ++ src/vnode/src/vnodeRead.c | 8 ++++++-- src/vnode/src/vnodeWrite.c | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 8ec66316e3..21d4480a73 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -124,8 +124,6 @@ int32_t vnodeDrop(int32_t vgId) { vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; - // remove from hash, so new messages wont be consumed - vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 32f9532138..62eb4dadcc 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -118,6 +118,8 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { + vnodeRemoveFromHash(pVnode); + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index b60fc5a8cd..2f8da0012b 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,14 +117,18 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + + if (pVnode->dropped) { + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } - SVnodeObj *pVnode = vparam; - int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 36516d81df..1b6e2e37ca 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -386,4 +386,6 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } + + taosMsleep(1000 * 3); } From 70213e65261e6ad7290d497383c7580efe93d16a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 11:55:49 +0800 Subject: [PATCH 11/25] test/sim: fix null to 0 in show dnodes; --- tests/script/unique/dnode/remove2.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 1d707bc4a3..11f6762129 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -162,7 +162,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show4 endi -if $data2_2 != null then +if $data2_2 != 0 then goto show4 endi if $data2_3 != 3 then From a07f6137bfeec37b24a179fa016457b027b472c8 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 14 May 2021 05:15:34 +0000 Subject: [PATCH 12/25] fix error --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index dd90a0440e..f7ee4e976f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -161,7 +161,7 @@ steps: - name: build image: ansible/centos7-ansible commands: - - yum install gcc cmake build-essential -y + - yum install -y gcc gcc-c++ make cmake - mkdir debug - cd debug - cmake .. From 226f86c8bf4d155f1ecc06ba196b5af95e3eb9e7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 15:09:49 +0800 Subject: [PATCH 13/25] [TD-4081]: [v3] fix vnode closing --- src/vnode/src/vnodeMain.c | 6 ++++++ src/vnode/src/vnodeMgmt.c | 2 -- src/vnode/src/vnodeStatus.c | 3 +++ src/vnode/src/vnodeSync.c | 11 +++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 21d4480a73..58baefefd2 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,6 +120,10 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } + if (pVnode->dropped) { + vnodeRelease(pVnode); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; @@ -473,6 +477,8 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); + vnodeRemoveFromHash(pVnode); + // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 62eb4dadcc..32f9532138 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -118,8 +118,6 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { - vnodeRemoveFromHash(pVnode); - vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index c482d1fd1a..1eaddc3d25 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,6 +66,9 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + if (pVnode->status == TAOS_VN_STATUS_CLOSING) + return true; + while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 05af34a34f..e5a1964915 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,6 +55,11 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while notify role", vgId); + vnodeRelease(pVnode); + return; + } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -75,6 +80,11 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); + vnodeRelease(pVnode); + return; + } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -129,6 +139,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); + vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } From 61da9bbe84ec017a8c2768e06add74713d02594c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 May 2021 15:18:05 +0800 Subject: [PATCH 14/25] TD-4100 --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2ec508f050..75312a0b03 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1471,7 +1471,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { * only the compressed size is less than the value of contLen - overhead, the compression is applied * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message */ - if (compLen < contLen - overhead) { + if (compLen > 0 && compLen < contLen - overhead) { SRpcComp *pComp = (SRpcComp *)pCont; pComp->reserved = 0; pComp->contLen = htonl(contLen); From 4fd37ea2a1beae14ebfc636fdcdf53171d3181bc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 12:43:08 +0800 Subject: [PATCH 15/25] [TD-4081]: vnode not close if dropped --- src/vnode/src/vnodeMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 58baefefd2..f02a5bedb0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -392,6 +392,10 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; + if (pVnode->dropped) { + vnodeRelease(pVnode); + return 0; + } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); From 198485c6f43667a8d14b4dfe7dfeeaf93d12307d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 15:32:31 +0800 Subject: [PATCH 16/25] Revert "test/sim: fix null to 0 in show dnodes;" This reverts commit 70213e65261e6ad7290d497383c7580efe93d16a. --- tests/script/unique/dnode/remove2.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 11f6762129..1d707bc4a3 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -162,7 +162,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show4 endi -if $data2_2 != 0 then +if $data2_2 != null then goto show4 endi if $data2_3 != 3 then From 7c93c4217a84a951dde190053abfa4364821b15f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 15:33:19 +0800 Subject: [PATCH 17/25] vnode/write: make last write msg to be written or confirmed --- src/vnode/src/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 1b6e2e37ca..3d16ae1faf 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(1000 * 3); + taosMsleep(1000); } From cd931e106a021221fed0f22a6575337dffb7c76a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 May 2021 16:29:09 +0800 Subject: [PATCH 18/25] Hotfix/sangshuduo/td 3985 taosdemo timestamp step overflow (#6131) * [TD-3985]: taosdemo timestamp step overflow. * fix specified subscribe test. * replace potential overflow of int32 to int64 * replace potential int32 overflow variables to int64. * replace potential int32 overflow variables to int64. * change max of int16 to int64 after type changed. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 70 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 72044753d8..7935546719 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -283,7 +283,7 @@ typedef struct SSuperTable_S { typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; - int32_t ntables; + int64_t ntables; int32_t vgroups; int16_t replica; int16_t quorum; @@ -412,7 +412,7 @@ typedef struct SThreadInfo_S { char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; - uint64_t ntables; + int64_t ntables; uint64_t data_of_rate; int64_t start_time; char* cols; @@ -581,7 +581,7 @@ SArguments g_args = { static SDbs g_Dbs; -static int g_totalChildTables = 0; +static uint64_t g_totalChildTables = 0; static SQueryMetaInfo g_queryInfo; static FILE * g_fpOfInsertResult = NULL; @@ -1879,7 +1879,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { formatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); - dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); @@ -1930,7 +1930,7 @@ static void printfDbInfoForQueryToFile( fprintf(fp, "================ database[%d] ================\n", index); fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); - fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables); fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); fprintf(fp, "replica: %d\n", dbInfos->replica); fprintf(fp, "quorum: %d\n", dbInfos->quorum); @@ -2884,7 +2884,7 @@ static void* createTable(void *sarg) } static int startMultiThreadCreateChildTable( - char* cols, int threads, uint64_t startFrom, uint64_t ntables, + char* cols, int threads, uint64_t startFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); @@ -2899,13 +2899,13 @@ static int startMultiThreadCreateChildTable( threads = 1; } - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; b = ntables % threads; for (int64_t i = 0; i < threads; i++) { @@ -2969,10 +2969,10 @@ static void createChildTables() { verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - int startFrom = 0; + uint64_t startFrom = 0; g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - verbosePrint("%s() LN%d: create %d child tables from %d\n", + verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n", __func__, __LINE__, g_totalChildTables, startFrom); startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, @@ -4756,7 +4756,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); } else { - verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRIu64" seq=%"PRIu64"\n", + verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, tableSeq); @@ -5123,14 +5123,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint64_t tableSeq = pThreadInfo->start_table_from; - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRIu64" insertRows=%"PRIu64"\n", + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); int64_t startTime = pThreadInfo->start_time; - assert(pThreadInfo->ntables > 0); - uint64_t batchPerTbl = interlaceRows; uint64_t batchPerTblTimes; @@ -5659,8 +5657,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - int ntables = 0; - int startFrom; + int64_t ntables = 0; + int64_t startFrom; if (superTblInfo) { int64_t limit; @@ -5730,13 +5728,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, taos_close(taos); - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -5892,7 +5890,7 @@ static void *readTable(void *sarg) { return NULL; } - int num_of_DPT; + uint64_t num_of_DPT; /* if (rinfo->superTblInfo) { num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; } else { @@ -5900,15 +5898,15 @@ static void *readTable(void *sarg) { num_of_DPT = g_args.num_of_DPT; // } - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); for (uint64_t j = 0; j < n; j++) { @@ -5940,7 +5938,7 @@ static void *readTable(void *sarg) { taos_free_result(pSql); } - fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", + fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); @@ -5962,17 +5960,17 @@ static void *readMetric(void *sarg) { return NULL; } - int num_of_DPT = rinfo->superTblInfo->insertRows; - int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int totalData = num_of_DPT * num_of_tables; + int64_t num_of_DPT = rinfo->superTblInfo->insertRows; + int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } - printf("%d records:\n", totalData); - fprintf(fp, "Querying On %d records:\n", totalData); + printf("%"PRId64" records:\n", totalData); + fprintf(fp, "Querying On %"PRId64" records:\n", totalData); for (int j = 0; j < n; j++) { char condition[COND_BUF_LEN] = "\0"; @@ -6070,11 +6068,11 @@ static int insertTestProcess() { end = taosGetTimestampMs(); if (g_totalChildTables > 0) { - fprintf(stderr, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); } } @@ -6400,16 +6398,16 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } @@ -6748,16 +6746,16 @@ static int subscribeTestProcess() { exit(-1); } - uint64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; - uint64_t a = ntables / threads; + int64_t a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - uint64_t b = 0; + int64_t b = 0; if (threads != 0) { b = ntables % threads; } From 2fb7c16b8757f5a1cf13bd8b5e008ead24aeafb4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 16:46:43 +0800 Subject: [PATCH 19/25] vnode/drop: ingore invalid vgroup id if already dropped --- src/vnode/src/vnodeMain.c | 2 +- src/vnode/src/vnodeWrite.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f02a5bedb0..ee28be3d2f 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -122,7 +122,7 @@ int32_t vnodeDrop(int32_t vgId) { } if (pVnode->dropped) { vnodeRelease(pVnode); - return TSDB_CODE_VND_INVALID_VGROUP_ID; + return TSDB_CODE_SUCCESS; } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 3d16ae1faf..7c67865a2b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(1000); + taosMsleep(2100); } From dae9262562cedbb6c6c61448f39342fb5f7e2b2e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 20:42:48 +0800 Subject: [PATCH 20/25] vnode/read: use app not ready instead of invalid vgroup id to make crash_gen happy --- src/vnode/src/vnodeRead.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2f8da0012b..5d344ebf11 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -118,9 +118,8 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; - if (pVnode->dropped) { - return TSDB_CODE_VND_INVALID_VGROUP_ID; + return TSDB_CODE_APP_NOT_READY; } SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); From 8b4a46de899acae5ecc89e204bfaf111bf074d90 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 May 2021 21:24:31 +0800 Subject: [PATCH 21/25] Hotfix/sangshuduo/td 3985 taosdemo timestamp step overflow (#6139) * [TD-3985]: taosdemo timestamp step overflow. * fix specified subscribe test. * replace potential overflow of int32 to int64 * replace potential int32 overflow variables to int64. * replace potential int32 overflow variables to int64. * change max of int16 to int64 after type changed. * improve more and more. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 68 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7935546719..12262f7a32 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -212,8 +212,8 @@ typedef struct SArguments_S { uint64_t interlace_rows; uint64_t num_of_RPR; // num_of_records_per_req uint64_t max_sql_len; - uint64_t num_of_tables; - uint64_t num_of_DPT; + int64_t num_of_tables; + int64_t num_of_DPT; int abort; int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision @@ -240,7 +240,7 @@ typedef struct SSuperTable_S { char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest int64_t childTblLimit; - uint64_t childTblOffset; + uint64_t childTblOffset; // int multiThreadWriteOneTbl; // 0: no, 1: yes uint64_t interlaceRows; // @@ -249,7 +249,7 @@ typedef struct SSuperTable_S { uint64_t maxSqlLen; // uint64_t insertInterval; // insert interval, will override global insert interval - uint64_t insertRows; + int64_t insertRows; int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json @@ -258,7 +258,7 @@ typedef struct SSuperTable_S { uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; - uint32_t tagCount; + uint32_t tagCount; StrColumn tags[MAX_TAG_COUNT]; char* childTblName; @@ -377,7 +377,7 @@ typedef struct SuperQueryInfo_S { int subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; - uint64_t childTblCount; + int64_t childTblCount; char childTblPrefix[MAX_TB_NAME_SIZE]; uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; @@ -581,7 +581,7 @@ SArguments g_args = { static SDbs g_Dbs; -static uint64_t g_totalChildTables = 0; +static int64_t g_totalChildTables = 0; static SQueryMetaInfo g_queryInfo; static FILE * g_fpOfInsertResult = NULL; @@ -1015,9 +1015,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->max_sql_len); printf("# Length of Binary: %d\n", arguments->len_of_binary); printf("# Number of Threads: %d\n", arguments->num_of_threads); - printf("# Number of Tables: %"PRIu64"\n", + printf("# Number of Tables: %"PRId64"\n", arguments->num_of_tables); - printf("# Number of Data per Table: %"PRIu64"\n", + printf("# Number of Data per Table: %"PRId64"\n", arguments->num_of_DPT); printf("# Database name: %s\n", arguments->database); printf("# Table prefix: %s\n", arguments->tb_prefix); @@ -1388,7 +1388,7 @@ static int printfInsertMeta() { printf(" childTblExists: \033[33m%s\033[0m\n", "error"); } - printf(" childTblCount: \033[33m%"PRIu64"\033[0m\n", + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); @@ -1404,7 +1404,7 @@ static int printfInsertMeta() { printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblOffset); } - printf(" insertRows: \033[33m%"PRIu64"\033[0m\n", + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); /* if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1571,7 +1571,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " childTblExists: %s\n", "error"); } - fprintf(fp, " childTblCount: %"PRIu64"\n", + fprintf(fp, " childTblCount: %"PRId64"\n", g_Dbs.db[i].superTbls[j].childTblCount); fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); @@ -1579,7 +1579,7 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRows: %"PRIu64"\n", + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); fprintf(fp, " interlace rows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); @@ -1697,7 +1697,7 @@ static void printfQueryMeta() { g_queryInfo.superQueryInfo.queryInterval); printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); - printf("childTblCount: \033[33m%"PRIu64"\033[0m\n", + printf("childTblCount: \033[33m%"PRId64"\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); @@ -2327,7 +2327,7 @@ static int calcRowLen(SSuperTable* superTbls) { static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, - uint64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { + int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { char command[BUFFER_SIZE] = "\0"; char limitBuf[100] = "\0"; @@ -2356,8 +2356,8 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, exit(-1); } - int childTblCount = (limit < 0)?10000:limit; - int count = 0; + int64_t childTblCount = (limit < 0)?10000:limit; + int64_t count = 0; if (childTblName == NULL) { childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (NULL == childTblName) { @@ -2404,7 +2404,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, - uint64_t* childTblCountOfSuperTbl) { + int64_t* childTblCountOfSuperTbl) { return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName, childTblNameOfSuperTbl, childTblCountOfSuperTbl, @@ -3000,7 +3000,7 @@ static void createChildTables() { snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); - verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRIu64" schema: %s\n", + verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", __func__, __LINE__, g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); startMultiThreadCreateChildTable( @@ -4772,7 +4772,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table static int64_t generateDataTail( SSuperTable* superTblInfo, uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { uint64_t len = 0; uint32_t ncols_per_record = 1; // count first col ts @@ -4948,7 +4948,7 @@ static int64_t generateInterlaceDataBuffer( char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, - uint64_t insertRows, + int64_t insertRows, int64_t startTime, uint64_t *pRemainderBufLen) { @@ -5008,7 +5008,7 @@ static int64_t generateProgressiveDataBuffer( int64_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, - int64_t startFrom, int64_t startTime, int64_t *pSamplePos, + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5061,7 +5061,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); - uint64_t insertRows; + int64_t insertRows; uint64_t interlaceRows; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5341,7 +5341,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { tableSeq ++) { int64_t start_time = pThreadInfo->start_time; - uint64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -5658,7 +5658,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } int64_t ntables = 0; - int64_t startFrom; + uint64_t startFrom; if (superTblInfo) { int64_t limit; @@ -5714,7 +5714,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - uint64_t childTblCount; + int64_t childTblCount; getChildNameOfSuperTableWithLimitAndOffset( taos, db_name, superTblInfo->sTblName, @@ -5890,7 +5890,7 @@ static void *readTable(void *sarg) { return NULL; } - uint64_t num_of_DPT; + int64_t num_of_DPT; /* if (rinfo->superTblInfo) { num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; } else { @@ -5909,11 +5909,11 @@ static void *readTable(void *sarg) { printf("%"PRId64" records:\n", totalData); fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); - for (uint64_t j = 0; j < n; j++) { + for (int j = 0; j < n; j++) { double totalT = 0; uint64_t count = 0; - for (uint64_t i = 0; i < num_of_tables; i++) { - sprintf(command, "select %s from %s%"PRIu64" where ts>= %" PRIu64, + for (int64_t i = 0; i < num_of_tables; i++) { + sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, aggreFunc[j], tb_prefix, i, sTime); double t = taosGetTimestampMs(); @@ -5976,13 +5976,13 @@ static void *readMetric(void *sarg) { char condition[COND_BUF_LEN] = "\0"; char tempS[64] = "\0"; - int m = 10 < num_of_tables ? 10 : num_of_tables; + int64_t m = 10 < num_of_tables ? 10 : num_of_tables; - for (int i = 1; i <= m; i++) { + for (int64_t i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "t1 = %d", i); + sprintf(tempS, "t1 = %"PRId64"", i); } else { - sprintf(tempS, " or t1 = %d ", i); + sprintf(tempS, " or t1 = %"PRId64" ", i); } strncat(condition, tempS, COND_BUF_LEN - 1); From 610603478155d87fa71f5c2d2fa97bb6b41e0f96 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 16 May 2021 11:24:35 +0800 Subject: [PATCH 22/25] vnode/close: wait write 900ms --- src/vnode/src/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 7c67865a2b..56ea32ccc0 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(2100); + taosMsleep(900); } From e620e2e83a26c8e8784d203010ef914172607a95 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 12:53:35 +0800 Subject: [PATCH 23/25] [td-4209]: add the uid check for super table metadata that is already cached in local buffer. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 ++++-- src/client/src/tscSchemaUtil.c | 1 + src/client/src/tscServer.c | 28 +++++++++++++--------------- src/client/src/tscUtil.c | 21 ++++++++++----------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b80dcfb577..47b2865313 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -298,7 +298,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d94d270a7..4bfd3bc88f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -67,14 +67,16 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name + char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name + uint64_t suid; // super table id } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name + uint64_t suid; // super table id int16_t sversion; int16_t tversion; STableComInfo tableInfo; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2ea382132b..114fc8ee73 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,6 +94,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 33325c7fd7..664f2989a8 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1920,13 +1920,13 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2537,19 +2537,16 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - } else { - //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; } + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2565,8 +2562,9 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { + // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3d2cf44560..89749202ef 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2941,22 +2941,25 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + cMeta->suid = pTableMeta->suid; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); -// uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = buf;//calloc(1, size); - + STableMeta* p = buf; taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); - if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + + // tableMeta exists, build child table meta according to the super table meta + // the uid need to be checked in addition to the general name of the super table. + if (p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -2964,13 +2967,9 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - -// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - -// tfree(p); return -1; } } From 07aed4fa708386e96b441ea1becfb89af80bea50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 15:23:44 +0800 Subject: [PATCH 24/25] [td-4209] --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 664f2989a8..0e16369cad 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1925,7 +1925,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->suid = pMetaMsg->suid; pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); From c824330bb5548f963f27255799c7f73d10765dcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 15:51:49 +0800 Subject: [PATCH 25/25] [td-225] refactor a sim script. --- tests/script/general/parser/commit.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index dfe521b92b..7c4c883fb1 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum, actual: $data00 + print expect $rowNum , actual: $data00 return -1 endi $i = $i + 1