From 95139cf4a1c03cd6f9bb60fe0daca48bbd84e4e9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 21:36:48 +0800 Subject: [PATCH 1/2] [TD-4130]: taosdemo subscribe super table. (#6086) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 134 +++++++++++++++++------------------- 1 file changed, 63 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a3a8968017..8866bf2607 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,12 +68,6 @@ enum TEST_MODE { INVAID_TEST }; -enum QUERY_MODE { - SYNC_QUERY_MODE, // 0 - ASYNC_QUERY_MODE, // 1 - INVALID_MODE -}; - #define MAX_RECORDS_PER_REQ 32766 #define MAX_SQL_SIZE 65536 @@ -1107,6 +1101,7 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile) } } + fprintf(fp, "%s", resultBuf); tmfclose(fp); } @@ -1142,6 +1137,7 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } @@ -6517,59 +6513,63 @@ static void *superSubscribe(void *sarg) { return NULL; } - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) { - // taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); + char topic[32] = {0}; + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); + replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - tsub[i] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); - if (NULL == tsub[i]) { + + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", + __func__, __LINE__, subSeq, subSqlstr); + tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); + if (NULL == tsub[subSeq]) { taos_close(pThreadInfo->taos); return NULL; } } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while(0); + } // start loop to consume result TAOS_RES* res = NULL; while(1) { - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } + for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + continue; + } - res = taos_consume(tsub[i]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taosMsleep(100); // ms + res = taos_consume(tsub[subSeq]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); + appendResultToFile(res, tmpFile); + } } } } } taos_free_result(res); - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taos_unsubscribe(tsub[subSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + } } taos_close(pThreadInfo->taos); @@ -6607,17 +6607,8 @@ static void *specifiedSubscribe(void *sarg) { return NULL; } - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) { - // taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms - // //printf("========sleep duration:%"PRIu64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { sprintf(topic, "taosdemo-subscribe-%d", i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { @@ -6630,11 +6621,7 @@ static void *specifiedSubscribe(void *sarg) { taos_close(pThreadInfo->taos); return NULL; } - } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while(0); - + } // start loop to consume result TAOS_RES* res = NULL; while(1) { @@ -6643,6 +6630,7 @@ static void *specifiedSubscribe(void *sarg) { continue; } + taosMsleep(1000); // ms res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; @@ -6699,31 +6687,35 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query from super table - if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) || - (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + //==== create sub threads for query for specified table + if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount, - g_queryInfo.specifiedQueryInfo.concurrent); - exit(-1); - } + g_queryInfo.specifiedQueryInfo.sqlCount); + } else { + if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { + errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + exit(-1); + } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - exit(-1); - } + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); + exit(-1); + } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + } } - //==== create sub threads for query from sub table + //==== create sub threads for super table query pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; if ((g_queryInfo.superQueryInfo.sqlCount > 0) From 5131e7b12a664ddcdfc48a1d11125d71f92cf64a Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Wed, 12 May 2021 09:41:03 +0800 Subject: [PATCH 2/2] Fix/td 3986 (#6088) * [TD-3986]: fix subscribe test case error * change jdbc version number --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 15 ++----- .../java/com/taosdata/jdbc/TSDBResultSet.java | 13 +++--- .../java/com/taosdata/jdbc/SubscribeTest.java | 44 +++++++++++++------ 7 files changed, 45 insertions(+), 35 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 9e325531d5..f8b3b7c3c6 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.28-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.29.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index de4b8f6bfb..61e976cb18 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.28-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.29.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index a31796ffde..968a9bf470 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.28 + 2.0.29 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 3400a82e73..d94d28d9fa 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.28 + 2.0.29 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 5e3ffffa4f..7f0cf7de8d 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -32,7 +32,7 @@ public class TSDBJNIConnector { // Connection pointer used in C private long taos = TSDBConstants.JNI_NULL_POINTER; // result set status in current connection - private boolean isResultsetClosed = true; + private boolean isResultsetClosed; private int affectedRows = -1; static { @@ -135,6 +135,7 @@ public class TSDBJNIConnector { // Try retrieving result set for the executed SQL using the current connection pointer. pSql = this.getResultSetImp(this.taos, pSql); + // if pSql == 0L that means resultset is closed isResultsetClosed = (pSql == TSDBConstants.JNI_NULL_POINTER); return pSql; @@ -172,16 +173,7 @@ public class TSDBJNIConnector { * Free resultset operation from C to release resultset pointer by JNI */ public int freeResultSet(long pSql) { - int res = TSDBConstants.JNI_SUCCESS; -// if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { -// throw new RuntimeException("Invalid result set pointer"); -// } - -// if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, pSql); -// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; -// } - + int res = this.freeResultSetImp(this.taos, pSql); isResultsetClosed = true; return res; } @@ -199,7 +191,6 @@ public class TSDBJNIConnector { // } // return resCode; // } - private native int freeResultSetImp(long connection, long result); /** diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 2576a25f0d..aba29d602b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -109,6 +109,8 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { public void close() throws SQLException { if (isClosed) return; + if (this.statement == null) + return; if (this.jniConnector != null) { int code = this.jniConnector.freeResultSet(this.resultSetPointer); if (code == TSDBConstants.JNI_CONNECTION_NULL) { @@ -461,12 +463,13 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { } public boolean isClosed() throws SQLException { - if (isClosed) - return true; - if (jniConnector != null) { - isClosed = jniConnector.isResultsetClosed(); - } return isClosed; +// if (isClosed) +// return true; +// if (jniConnector != null) { +// isClosed = jniConnector.isResultsetClosed(); +// } +// return isClosed; } public String getNString(int columnIndex) throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 3a223ed981..24c73fdd5c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -19,6 +20,7 @@ public class SubscribeTest { String tName = "t0"; String host = "127.0.0.1"; String topic = "test"; + private long ts; @Test public void subscribe() { @@ -27,26 +29,40 @@ public class SubscribeTest { TSDBConnection conn = connection.unwrap(TSDBConnection.class); TSDBSubscribe subscribe = conn.subscribe(topic, rawSql, false); - int a = 0; - while (true) { - TimeUnit.MILLISECONDS.sleep(1000); + for (int j = 0; j < 10; j++) { + TimeUnit.SECONDS.sleep(1); TSDBResultSet resSet = subscribe.consume(); + + int rowCnt = 0; while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); + if (rowCnt == 0) { + long cur_ts = resSet.getTimestamp(1).getTime(); + int k = resSet.getInt(2); + int v = resSet.getInt(3); + Assert.assertEquals(ts, cur_ts); + Assert.assertEquals(100, k); + Assert.assertEquals(1, v); } - System.out.println("\n======" + a + "=========="); - } - a++; - if (a >= 2) { - break; + if (rowCnt == 1) { + long cur_ts = resSet.getTimestamp(1).getTime(); + int k = resSet.getInt(2); + int v = resSet.getInt(3); + Assert.assertEquals(ts + 1, cur_ts); + Assert.assertEquals(101, k); + Assert.assertEquals(2, v); + + } + rowCnt++; } + if (j == 0) + Assert.assertEquals(2, rowCnt); resSet.close(); } - subscribe.close(true); - } catch (Exception e) { - e.printStackTrace(); + + + } catch (SQLException | InterruptedException throwables) { + throwables.printStackTrace(); } } @@ -62,7 +78,7 @@ public class SubscribeTest { statement.execute("drop database if exists " + dbName); statement.execute("create database if not exists " + dbName); statement.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - long ts = System.currentTimeMillis(); + ts = System.currentTimeMillis(); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 100, 1)"); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + (ts + 1) + ", 101, 2)"); }