diff --git a/cmake/platform.inc b/cmake/platform.inc index dcd0183e27..5f7391c996 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -102,6 +102,12 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LINUX TRUE) SET(TD_LINUX_64 FALSE) SET(TD_ARM_64 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + SET(CPUTYPE "mips64") + MESSAGE(STATUS "Set CPUTYPE to mips64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_MIPS_64 TRUE) ENDIF () ELSE () diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 56d595ff1f..0348606d6b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -307,7 +307,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bf41449e13..81443360f4 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -68,14 +68,16 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name + char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name + uint64_t suid; // super table id } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name + uint64_t suid; // super table id int16_t sversion; int16_t tversion; STableComInfo tableInfo; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2ea382132b..114fc8ee73 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,6 +94,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6587a55152..9b7d5c0c7f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1828,13 +1828,13 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->suid = pMetaMsg->suid; + pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2448,19 +2448,16 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - } else { - //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; } + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2476,8 +2473,9 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { + // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a698e239df..37e836dc1c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3370,22 +3370,25 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + cMeta->suid = pTableMeta->suid; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); -// uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = buf;//calloc(1, size); - + STableMeta* p = buf; taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); - if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + + // tableMeta exists, build child table meta according to the super table meta + // the uid need to be checked in addition to the general name of the super table. + if (p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -3393,13 +3396,9 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - -// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - -// tfree(p); return -1; } } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 2f4aa4c2b2..e07c3611d7 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -142,12 +142,15 @@ extern int32_t tsMonitorInterval; extern int8_t tsEnableStream; // internal +extern int8_t tsCompactMnodeWal; extern int8_t tsPrintAuth; extern int8_t tscEmbedded; extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; extern char tsMnodeDir[]; +extern char tsMnodeBakDir[]; +extern char tsMnodeTmpDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index db97c3a5af..d6bbc288ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -176,12 +176,15 @@ int32_t tsMonitorInterval = 30; // seconds int8_t tsEnableStream = 1; // internal +int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; int8_t tscEmbedded = 0; char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeTmpDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeBakDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 8804cc5da0..52858e7f88 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -207,10 +207,69 @@ public class TSDBPreparedStatementTest { while(rs.next()) { rows++; } - Assert.assertEquals(numOfRows, rows); + Assert.assertEquals(numOfRows, rows); } } + @Test + public void bindDataSelectColumnTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++){ + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? (ts, f1, f7) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s2.add(null); + }else{ + s2.add("分支" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s5 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s5.add(null); + }else{ + s5.add("test" + i % 10); + } + } + s.setString(2, s5, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + + @Test public void setBoolean() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 43a08a800a..f3961e3787 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -9,7 +9,7 @@ const ffi = require('ffi-napi'); const ArrayType = require('ref-array-napi'); const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); -const errors = require ('./error'); +const errors = require('./error'); const TaosObjects = require('./taosobjects'); const { NULL_POINTER } = require('ref-napi'); @@ -22,7 +22,7 @@ function convertMicrosecondsToDatetime(time) { return new TaosObjects.TaosTimestamp(time * 0.001, true); } -function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { timestampConverter = convertMillisecondsToDatetime; if (micro == true) { timestampConverter = convertMicrosecondsToDatetime; @@ -44,14 +44,14 @@ function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false } return res; } -function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = new Array(data.length); for (let i = 0; i < data.length; i++) { if (data[i] == 0) { res[i] = false; } - else if (data[i] == 1){ + else if (data[i] == 1) { res[i] = true; } else if (data[i] == FieldTypes.C_BOOL_NULL) { @@ -60,29 +60,29 @@ function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,1); + let d = data.readIntLE(currOffset, 1); res.push(d == FieldTypes.C_TINYINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,2); + let d = data.readIntLE(currOffset, 2); res.push(d == FieldTypes.C_SMALLINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -93,7 +93,7 @@ function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -104,7 +104,7 @@ function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -115,7 +115,7 @@ function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -126,7 +126,7 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -142,7 +142,7 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; @@ -153,23 +153,23 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { // Object with all the relevant converters from pblock data to javascript readable data let convertFunctions = { - [FieldTypes.C_BOOL] : convertBool, - [FieldTypes.C_TINYINT] : convertTinyint, - [FieldTypes.C_SMALLINT] : convertSmallint, - [FieldTypes.C_INT] : convertInt, - [FieldTypes.C_BIGINT] : convertBigint, - [FieldTypes.C_FLOAT] : convertFloat, - [FieldTypes.C_DOUBLE] : convertDouble, - [FieldTypes.C_BINARY] : convertBinary, - [FieldTypes.C_TIMESTAMP] : convertTimestamp, - [FieldTypes.C_NCHAR] : convertNchar + [FieldTypes.C_BOOL]: convertBool, + [FieldTypes.C_TINYINT]: convertTinyint, + [FieldTypes.C_SMALLINT]: convertSmallint, + [FieldTypes.C_INT]: convertInt, + [FieldTypes.C_BIGINT]: convertBigint, + [FieldTypes.C_FLOAT]: convertFloat, + [FieldTypes.C_DOUBLE]: convertDouble, + [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_TIMESTAMP]: convertTimestamp, + [FieldTypes.C_NCHAR]: convertNchar } // Define TaosField structure var char_arr = ArrayType(ref.types.char); var TaosField = Struct({ - 'name': char_arr, - }); + 'name': char_arr, +}); TaosField.fields.name.type.size = 65; TaosField.defineProperty('type', ref.types.char); TaosField.defineProperty('bytes', ref.types.short); @@ -183,7 +183,7 @@ TaosField.defineProperty('bytes', ref.types.short); * @classdesc The CTaosInterface is the interface through which Node.JS communicates data back and forth with TDengine. It is not advised to * access this class directly and use it unless you understand what these functions do. */ -function CTaosInterface (config = null, pass = false) { +function CTaosInterface(config = null, pass = false) { ref.types.char_ptr = ref.refType(ref.types.char); ref.types.void_ptr = ref.refType(ref.types.void); ref.types.void_ptr2 = ref.refType(ref.types.void_ptr); @@ -196,64 +196,65 @@ function CTaosInterface (config = null, pass = false) { taoslibname = 'libtaos'; } this.libtaos = ffi.Library(taoslibname, { - 'taos_options': [ ref.types.int, [ ref.types.int , ref.types.void_ptr ] ], - 'taos_init': [ ref.types.void, [ ] ], + 'taos_options': [ref.types.int, [ref.types.int, ref.types.void_ptr]], + 'taos_init': [ref.types.void, []], //TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) - 'taos_connect': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int ] ], + 'taos_connect': [ref.types.void_ptr, [ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int]], //void taos_close(TAOS *taos) - 'taos_close': [ ref.types.void, [ ref.types.void_ptr ] ], - //int *taos_fetch_lengths(TAOS_RES *taos); - 'taos_fetch_lengths': [ ref.types.void_ptr, [ ref.types.void_ptr ] ], + 'taos_close': [ref.types.void, [ref.types.void_ptr]], + //int *taos_fetch_lengths(TAOS_RES *res); + 'taos_fetch_lengths': [ref.types.void_ptr, [ref.types.void_ptr]], //int taos_query(TAOS *taos, char *sqlstr) - 'taos_query': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr ] ], - //int taos_affected_rows(TAOS *taos) - 'taos_affected_rows': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_query': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr]], + //int taos_affected_rows(TAOS_RES *res) + 'taos_affected_rows': [ref.types.int, [ref.types.void_ptr]], //int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) - 'taos_fetch_block': [ ref.types.int, [ ref.types.void_ptr, ref.types.void_ptr] ], + 'taos_fetch_block': [ref.types.int, [ref.types.void_ptr, ref.types.void_ptr]], //int taos_num_fields(TAOS_RES *res); - 'taos_num_fields': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_num_fields': [ref.types.int, [ref.types.void_ptr]], //TAOS_ROW taos_fetch_row(TAOS_RES *res) //TAOS_ROW is void **, but we set the return type as a reference instead to get the row - 'taos_fetch_row': [ ref.refType(ref.types.void_ptr2), [ ref.types.void_ptr ] ], + 'taos_fetch_row': [ref.refType(ref.types.void_ptr2), [ref.types.void_ptr]], + 'taos_print_row': [ref.types.int, [ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], //int taos_result_precision(TAOS_RES *res) - 'taos_result_precision': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_result_precision': [ref.types.int, [ref.types.void_ptr]], //void taos_free_result(TAOS_RES *res) - 'taos_free_result': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_free_result': [ref.types.void, [ref.types.void_ptr]], //int taos_field_count(TAOS *taos) - 'taos_field_count': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_field_count': [ref.types.int, [ref.types.void_ptr]], //TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) - 'taos_fetch_fields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ], + 'taos_fetch_fields': [ref.refType(TaosField), [ref.types.void_ptr]], //int taos_errno(TAOS *taos) - 'taos_errno': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_errno': [ref.types.int, [ref.types.void_ptr]], //char *taos_errstr(TAOS *taos) - 'taos_errstr': [ ref.types.char_ptr, [ ref.types.void_ptr] ], + 'taos_errstr': [ref.types.char_ptr, [ref.types.void_ptr]], //void taos_stop_query(TAOS_RES *res); - 'taos_stop_query': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_stop_query': [ref.types.void, [ref.types.void_ptr]], //char *taos_get_server_info(TAOS *taos); - 'taos_get_server_info': [ ref.types.char_ptr, [ ref.types.void_ptr ] ], + 'taos_get_server_info': [ref.types.char_ptr, [ref.types.void_ptr]], //char *taos_get_client_info(); - 'taos_get_client_info': [ ref.types.char_ptr, [ ] ], + 'taos_get_client_info': [ref.types.char_ptr, []], // ASYNC // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) - 'taos_query_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_query_a': [ref.types.void, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr]], // void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); - 'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]], + 'taos_fetch_rows_a': [ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr]], // Subscription //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) - 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ], + 'taos_subscribe': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], // TAOS_RES *taos_consume(TAOS_SUB *tsub) - 'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ], + 'taos_consume': [ref.types.void_ptr, [ref.types.void_ptr]], //void taos_unsubscribe(TAOS_SUB *tsub); - 'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ], + 'taos_unsubscribe': [ref.types.void, [ref.types.void_ptr]], // Continuous Query //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), // int64_t stime, void *param, void (*callback)(void *)); - 'taos_open_stream': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_open_stream': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr]], //void taos_close_stream(TAOS_STREAM *tstr); - 'taos_close_stream': [ ref.types.void, [ ref.types.void_ptr ] ] + 'taos_close_stream': [ref.types.void, [ref.types.void_ptr]] }); if (pass == false) { @@ -264,7 +265,7 @@ function CTaosInterface (config = null, pass = false) { try { this._config = ref.allocCString(config); } - catch(err){ + catch (err) { throw "Attribute Error: config is expected as a str"; } } @@ -276,38 +277,38 @@ function CTaosInterface (config = null, pass = false) { return this; } CTaosInterface.prototype.config = function config() { - return this._config; - } -CTaosInterface.prototype.connect = function connect(host=null, user="root", password="taosdata", db=null, port=0) { - let _host,_user,_password,_db,_port; - try { + return this._config; +} +CTaosInterface.prototype.connect = function connect(host = null, user = "root", password = "taosdata", db = null, port = 0) { + let _host, _user, _password, _db, _port; + try { _host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: host is expected as a str"; } try { _user = ref.allocCString(user) } - catch(err) { + catch (err) { throw "Attribute Error: user is expected as a str"; } try { _password = ref.allocCString(password); } - catch(err) { + catch (err) { throw "Attribute Error: password is expected as a str"; } try { _db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: db is expected as a str"; } try { _port = ref.alloc(ref.types.int, port); } - catch(err) { + catch (err) { throw TypeError("port is expected as an int") } let connection = this.libtaos.taos_connect(_host, _user, _password, _db, _port); @@ -324,10 +325,10 @@ CTaosInterface.prototype.close = function close(connection) { console.log("Connection is closed"); } CTaosInterface.prototype.query = function query(connection, sql) { - return this.libtaos.taos_query(connection, ref.allocCString(sql)); + return this.libtaos.taos_query(connection, ref.allocCString(sql)); } -CTaosInterface.prototype.affectedRows = function affectedRows(connection) { - return this.libtaos.taos_affected_rows(connection); +CTaosInterface.prototype.affectedRows = function affectedRows(result) { + return this.libtaos.taos_affected_rows(result); } CTaosInterface.prototype.useResult = function useResult(result) { @@ -337,8 +338,8 @@ CTaosInterface.prototype.useResult = function useResult(result) { pfields = ref.reinterpret(pfields, this.fieldsCount(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -347,11 +348,10 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let pblock = this.libtaos.taos_fetch_row(result); - let num_of_rows = 1; - if (ref.isNull(pblock) == true) { - return {block:null, num_of_rows:0}; + let pblock = ref.NULL_POINTER; + let num_of_rows = this.libtaos.taos_fetch_block(result, pblock); + if (ref.isNull(pblock.deref()) == true) { + return { block: null, num_of_rows: 0 }; } var fieldL = this.libtaos.taos_fetch_lengths(result); @@ -359,10 +359,10 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); var fieldlens = []; - + if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 4, i * 4); let len = plen.readInt32LE(0); fieldlens.push(len); } @@ -370,21 +370,23 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let blocks = new Array(fields.length); blocks.fill(null); - //num_of_rows = Math.abs(num_of_rows); + num_of_rows = Math.abs(num_of_rows); let offset = 0; + let ptr = pblock.deref(); + for (let i = 0; i < fields.length; i++) { - pdata = ref.reinterpret(pblock,8,i*8); - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); - } + pdata = ref.reinterpret(ptr, 8, i * 8); + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, isMicro); + } } - return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} + return { blocks: blocks, num_of_rows } } CTaosInterface.prototype.fetchRow = function fetchRow(result, fields) { let row = this.libtaos.taos_fetch_row(result); @@ -414,7 +416,7 @@ CTaosInterface.prototype.errStr = function errStr(result) { // Async CTaosInterface.prototype.query_a = function query_a(connection, sql, callback, param = ref.ref(ref.NULL)) { // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, int), void *param) - callback = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], callback); + callback = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], callback); this.libtaos.taos_query_a(connection, ref.allocCString(sql), callback, param); return param; } @@ -439,46 +441,46 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, var fieldL = cti.libtaos.taos_fetch_lengths(result); var fieldlens = []; if (ref.isNull(fieldL) == false) { - - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 8, i*8); - let len = ref.get(plen,0,ref.types.int32); + + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 8, i * 8); + let len = ref.get(plen, 0, ref.types.int32); fieldlens.push(len); } } - if (numOfRows2 > 0){ + if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; - } + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row, 8, i * 8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], asyncCallbackWrapper); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], asyncCallbackWrapper); this.libtaos.taos_fetch_rows_a(result, asyncCallbackWrapper, param); return param; } // Fetch field meta data by result handle -CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { +CTaosInterface.prototype.fetchFields_a = function fetchFields_a(result) { let pfields = this.fetchFields(result); let pfieldscount = this.numFields(result); let fields = []; if (ref.isNull(pfields) == false) { - pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0); + pfields = ref.reinterpret(pfields, 68 * pfieldscount, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 64 = name //65 = type, 66 - 67 = bytes - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -488,7 +490,7 @@ CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { } // Stop a query by result handle CTaosInterface.prototype.stopQuery = function stopQuery(result) { - if (result != null){ + if (result != null) { this.libtaos.taos_stop_query(result); } else { @@ -509,13 +511,13 @@ CTaosInterface.prototype.subscribe = function subscribe(connection, restart, top try { sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: sql is expected as a str"; } try { topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw TypeError("topic is expected as a str"); } @@ -539,8 +541,8 @@ CTaosInterface.prototype.consume = function consume(subscription) { pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,64,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 64, i)), bytes: pfields[i + 64], type: pfields[i + 66] }) @@ -548,7 +550,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { } let data = []; - while(true) { + while (true) { let { blocks, num_of_rows } = this.fetchBlock(result, fields); if (num_of_rows == 0) { break; @@ -559,7 +561,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { for (let j = 0; j < fields.length; j++) { rowBlock[j] = blocks[j][i]; } - data[data.length-1] = (rowBlock); + data[data.length - 1] = (rowBlock); } } return { data: data, fields: fields, result: result }; @@ -570,11 +572,11 @@ CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) { } // Continuous Query -CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime,stoppingCallback, param = ref.ref(ref.NULL)) { +CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime, stoppingCallback, param = ref.ref(ref.NULL)) { try { sql = ref.allocCString(sql); } - catch(err) { + catch (err) { throw "Attribute Error: sql string is expected as a str"; } var cti = this; @@ -587,7 +589,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb let offset = 0; if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { + if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); @@ -596,8 +598,8 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb } callback(param2, result2, blocks, fields); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2) ], asyncCallbackWrapper); - asyncStoppingCallbackWrapper = ffi.Callback( ref.types.void, [ ref.types.void_ptr ], stoppingCallback); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2)], asyncCallbackWrapper); + asyncStoppingCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr], stoppingCallback); let streamHandle = this.libtaos.taos_open_stream(connection, sql, asyncCallbackWrapper, stime, param, asyncStoppingCallbackWrapper); if (ref.isNull(streamHandle)) { throw new errors.TDError('Failed to open a stream with TDengine'); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index e18e6c2500..f879d89d48 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,7 +1,7 @@ const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') -const errors = require ('./error') +const errors = require('./error') const TaosQuery = require('./taosquery') const { PerformanceObserver, performance } = require('perf_hooks'); module.exports = TDengineCursor; @@ -22,7 +22,7 @@ module.exports = TDengineCursor; * @property {fields} - Array of the field objects in order from left to right of the latest data retrieved * @since 1.0.0 */ -function TDengineCursor(connection=null) { +function TDengineCursor(connection = null) { //All parameters are store for sync queries only. this._rowcount = -1; this._connection = null; @@ -91,7 +91,7 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { callback = options; } if (typeof options != 'object') options = {} @@ -144,10 +144,10 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback } TDengineCursor.prototype._createAffectedResponse = function (num, time) { - return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype._createSetResponse = function (num, time) { - return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype.executemany = function executemany() { @@ -176,27 +176,22 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { throw new errors.OperationalError("Invalid use of fetchall, either result or fields from query are null. First execute a query first"); } - let data = []; + let num_of_rows = this._chandle.affectedRows(this._result); + let data = new Array(num_of_rows); + this._rowcount = 0; - //let nodetime = 0; + let time = 0; const obs = new PerformanceObserver((items) => { time += items.getEntries()[0].duration; performance.clearMarks(); }); - /* - const obs2 = new PerformanceObserver((items) => { - nodetime += items.getEntries()[0].duration; - performance.clearMarks(); - }); - obs2.observe({ entryTypes: ['measure'] }); - performance.mark('nodea'); - */ obs.observe({ entryTypes: ['measure'] }); performance.mark('A'); - while(true) { - + while (true) { let blockAndRows = this._chandle.fetchBlock(this._result, this._fields); + // console.log(blockAndRows); + // break; let block = blockAndRows.blocks; let num_of_rows = blockAndRows.num_of_rows; if (num_of_rows == 0) { @@ -205,22 +200,24 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { this._rowcount += num_of_rows; let numoffields = this._fields.length; for (let i = 0; i < num_of_rows; i++) { - data.push([]); - + // data.push([]); + let rowBlock = new Array(numoffields); for (let j = 0; j < numoffields; j++) { rowBlock[j] = block[j][i]; } - data[data.length-1] = (rowBlock); + data[this._rowcount - num_of_rows + i] = (rowBlock); + // data.push(rowBlock); } } + performance.mark('B'); performance.measure('query', 'A', 'B'); let response = this._createSetResponse(this._rowcount, time) console.log(response); - // this._connection._clearResultSet(); + // this._connection._clearResultSet(); let fields = this.fields; this._reset_result(); this.data = data; @@ -239,12 +236,12 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { * @return {number | Buffer} Number of affected rows or a Buffer that points to the results of the query * @since 1.0.0 */ -TDengineCursor.prototype.execute_a = function execute_a (operation, options, callback, param) { +TDengineCursor.prototype.execute_a = function execute_a(operation, options, callback, param) { if (operation == undefined) { throw new errors.ProgrammingError('No operation passed as argument'); return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -265,14 +262,14 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal } if (resCode >= 0) { -// let fieldCount = cr._chandle.numFields(res2); -// if (fieldCount == 0) { -// //cr._chandle.freeResult(res2); -// return res2; -// } -// else { -// return res2; -// } + // let fieldCount = cr._chandle.numFields(res2); + // if (fieldCount == 0) { + // //cr._chandle.freeResult(res2); + // return res2; + // } + // else { + // return res2; + // } return res2; } @@ -317,7 +314,7 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal * }) */ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callback, param = {}) { - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -360,17 +357,17 @@ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callb for (let k = 0; k < fields.length; k++) { rowBlock[k] = block[k][j]; } - data[data.length-1] = rowBlock; + data[data.length - 1] = rowBlock; } } cr._chandle.freeResult(result2); // free result, avoid seg faults and mem leaks! - callback(param2, result2, numOfRows2, {data:data,fields:fields}); + callback(param2, result2, numOfRows2, { data: data, fields: fields }); } } ref.writeObject(buf, 0, param); param = this._chandle.fetch_rows_a(result, asyncCallbackWrapper, buf); //returned param - return {param:param,result:result}; + return { param: param, result: result }; } /** * Stop a query given the result handle. @@ -428,7 +425,7 @@ TDengineCursor.prototype.subscribe = function subscribe(config) { */ TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) { while (true) { - let { data, fields, result} = this._chandle.consume(subscription); + let { data, fields, result } = this._chandle.consume(subscription); callback(data, fields, result); } } @@ -450,30 +447,30 @@ TDengineCursor.prototype.unsubscribe = function unsubscribe(subscription) { * @return {Buffer} A buffer pointing to the stream handle * @since 1.3.0 */ - TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { - let buf = ref.alloc('Object'); - ref.writeObject(buf, 0, param); +TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { + let buf = ref.alloc('Object'); + ref.writeObject(buf, 0, param); - let asyncCallbackWrapper = function (param2, result2, blocks, fields) { - let data = []; - let num_of_rows = blocks[0].length; - for (let j = 0; j < num_of_rows; j++) { - data.push([]); - let rowBlock = new Array(fields.length); - for (let k = 0; k < fields.length; k++) { - rowBlock[k] = blocks[k][j]; - } - data[data.length-1] = rowBlock; - } - callback(param2, result2, blocks, fields); - } - return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); - } - /** - * Close a stream - * @param {Buffer} - A buffer pointing to the handle of the stream to be closed - * @since 1.3.0 - */ - TDengineCursor.prototype.closeStream = function closeStream(stream) { - this._chandle.closeStream(stream); - } + let asyncCallbackWrapper = function (param2, result2, blocks, fields) { + let data = []; + let num_of_rows = blocks[0].length; + for (let j = 0; j < num_of_rows; j++) { + data.push([]); + let rowBlock = new Array(fields.length); + for (let k = 0; k < fields.length; k++) { + rowBlock[k] = blocks[k][j]; + } + data[data.length - 1] = rowBlock; + } + callback(param2, result2, blocks, fields); + } + return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); +} +/** + * Close a stream + * @param {Buffer} - A buffer pointing to the handle of the stream to be closed + * @since 1.3.0 + */ +TDengineCursor.prototype.closeStream = function closeStream(stream) { + this._chandle.closeStream(stream); +} diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json deleted file mode 100644 index 9ca174ccd1..0000000000 --- a/src/connector/nodejs/package-lock.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "name": "td2.0-connector", - "version": "2.0.6", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "array-index": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", - "integrity": "sha1-7FanSe4QPk4Ix5C5w1PfFgVbl/k=", - "requires": { - "debug": "^2.2.0", - "es6-symbol": "^3.0.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "requires": { - "ms": "2.1.2" - } - }, - "es5-ext": { - "version": "0.10.53", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", - "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", - "requires": { - "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.3", - "next-tick": "~1.0.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "requires": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "requires": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "ext": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", - "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", - "requires": { - "type": "^2.0.0" - }, - "dependencies": { - "type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", - "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" - } - } - }, - "ffi-napi": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", - "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", - "requires": { - "debug": "^4.1.1", - "get-uv-event-loop-napi-h": "^1.0.5", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1", - "ref-napi": "^2.0.1", - "ref-struct-di": "^1.1.0" - }, - "dependencies": { - "ref-napi": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", - "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - } - } - }, - "get-symbol-from-current-process-h": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", - "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" - }, - "get-uv-event-loop-napi-h": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", - "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", - "requires": { - "get-symbol-from-current-process-h": "^1.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" - }, - "node-addon-api": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", - "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" - }, - "node-gyp-build": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", - "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" - }, - "ref-array-napi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", - "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", - "requires": { - "array-index": "1", - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "ref-napi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", - "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - }, - "ref-struct-di": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", - "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, - "ref-struct-napi": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", - "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", - "requires": { - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - } - } -} diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index b39ce2c17d..d21b62108b 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.6", + "version": "2.0.7", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 410e6bb188..ee4dfcf852 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -40,6 +40,7 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" #include "module.h" +#include "mnode.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } @@ -216,6 +217,17 @@ static int32_t dnodeInitStorage() { sprintf(tsDnodeDir, "%s/dnode", tsDataDir); // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); + if (tsCompactMnodeWal == 1) { + sprintf(tsMnodeTmpDir, "%s/mnode_tmp", tsDataDir); + tfsRmdir(tsMnodeTmpDir); + if (dnodeCreateDir(tsMnodeTmpDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsMnodeTmpDir, strerror(errno)); + return -1; + } + + sprintf(tsMnodeBakDir, "%s/mnode_bak", tsDataDir); + //tfsRmdir(tsMnodeBakDir); + } //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index e49b3eba99..ee37ffdcbb 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,6 +42,8 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; + } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { + tsCompactMnodeWal = 1; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 2495a42ba2..203ac57469 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -73,6 +73,9 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mnodeCompactWal(); +int32_t mnodeCompactComponents(); + #ifdef __cplusplus } #endif diff --git a/src/inc/monitor.h b/src/inc/monitor.h index 1aefb0b848..d2e5e06487 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -54,7 +54,8 @@ void monCleanupSystem(); void monSaveAcctLog(SAcctMonitorObj *pMonObj); void monSaveLog(int32_t level, const char *const format, ...); void monExecuteSQL(char *sql); - +typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code); +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param); #ifdef __cplusplus } #endif diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ce6f7c4f22..d3ac0ce280 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -427,6 +427,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") +// monitor +#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") + #ifdef __cplusplus } #endif diff --git a/src/kit/taosdemo/subscribe.json b/src/kit/taosdemo/subscribe.json index 18846c8116..9faf03a03d 100644 --- a/src/kit/taosdemo/subscribe.json +++ b/src/kit/taosdemo/subscribe.json @@ -1,5 +1,5 @@ { - "filetype":"subscribe", + "filetype": "subscribe", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 068aa52283..e8075c69ba 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -61,6 +61,8 @@ extern char configDir[]; #define QUERY_JSON_NAME "query.json" #define SUBSCRIBE_JSON_NAME "subscribe.json" +#define STR_INSERT_INTO "INSERT INTO " + enum TEST_MODE { INSERT_TEST, // 0 QUERY_TEST, // 1 @@ -70,6 +72,8 @@ enum TEST_MODE { #define MAX_RECORDS_PER_REQ 32766 +#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. + #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -196,6 +200,8 @@ typedef struct { } SColDes; /* Used by main to communicate with parse_opt. */ +static char *g_dupstr = NULL; + typedef struct SArguments_S { char * metaFile; uint32_t test_mode; @@ -241,7 +247,7 @@ typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN + 1]; char dataType[MAX_TB_NAME_SIZE]; uint32_t dataLen; - char note[128]; + char note[128]; } StrColumn; typedef struct SSuperTable_S { @@ -374,7 +380,7 @@ typedef struct SpecifiedQueryInfo_S { uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms uint64_t queryTimes; - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; @@ -389,7 +395,7 @@ typedef struct SuperQueryInfo_S { uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; @@ -535,11 +541,12 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile); +static int postProceSql(char *host, struct sockaddr_in *pServAddr, + uint16_t port, char* sqlstr, char *resultFile); /* ************ Global variables ************ */ @@ -690,8 +697,9 @@ static void printHelp() { "The data_type of columns, default: INT,INT,INT,INT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, - "The number of columns per record. Default is 4."); + printf("%s%s%s%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is 4. Max values is ", + MAX_NUM_DATATYPE); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -735,7 +743,6 @@ static bool isStringNumber(char *input) } static void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { @@ -879,20 +886,31 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->database = argv[++i]; } else if (strcmp(argv[i], "-l") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); - exit(EXIT_FAILURE); + if (argc == i+1) { + if (!isStringNumber(argv[i+1])) { + printHelp(); + errorPrint("%s", "\n\t-l need a number following!\n"); + exit(EXIT_FAILURE); + } } arguments->num_of_CPR = atoi(argv[++i]); + + if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + prompt(); + arguments->num_of_CPR = MAX_NUM_DATATYPE; + } + + for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + arguments->datatype[col] = NULL; + } + } else if (strcmp(argv[i], "-b") == 0) { if (argc == i+1) { printHelp(); errorPrint("%s", "\n\t-b need valid string following!\n"); exit(EXIT_FAILURE); } - sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { // only one col @@ -909,12 +927,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[0] = argv[i]; + arguments->datatype[0] = argv[i]; } else { // more than one col int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; + g_dupstr = strdup(argv[i]); + char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { if (strcasecmp(token, "INT") @@ -927,16 +945,15 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { printHelp(); - free(dupstr); + free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[index++] = token; + arguments->datatype[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } - free(dupstr); - sptr[index] = NULL; + arguments->datatype[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { if ((argc == i+1) || @@ -1071,16 +1088,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } static bool getInfoFromJsonFile(char* file); -//static int generateOneRowDataForStb(SSuperTable* stbInfo); -//static int getDataIntoMemForStb(SSuperTable* stbInfo); static void init_rand_data(); static void tmfclose(FILE *fp) { if (NULL != fp) { @@ -1180,7 +1193,8 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } - verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", + __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } @@ -1333,13 +1347,15 @@ static int printfInsertMeta() { printf("interface: \033[33m%s\033[0m\n", (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt"); - printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("host: \033[33m%s:%u\033[0m\n", + g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); printf("configDir: \033[33m%s\033[0m\n", configDir); printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); - printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("thread num of create table: \033[33m%d\033[0m\n", + g_Dbs.threadCountByCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%"PRIu64"\033[0m\n", @@ -1351,7 +1367,8 @@ static int printfInsertMeta() { for (int i = 0; i < g_Dbs.dbCount; i++) { printf("database[\033[33m%d\033[0m]:\n", i); - printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); + printf(" database[%d] name: \033[33m%s\033[0m\n", + i, g_Dbs.db[i].dbName); if (0 == g_Dbs.db[i].drop) { printf(" drop: \033[33mno\033[0m\n"); } else { @@ -1359,40 +1376,51 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.blocks > 0) { - printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + printf(" blocks: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.blocks); } if (g_Dbs.db[i].dbCfg.cache > 0) { - printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + printf(" cache: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.cache); } if (g_Dbs.db[i].dbCfg.days > 0) { - printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + printf(" days: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.days); } if (g_Dbs.db[i].dbCfg.keep > 0) { - printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + printf(" keep: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.keep); } if (g_Dbs.db[i].dbCfg.replica > 0) { - printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + printf(" replica: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.replica); } if (g_Dbs.db[i].dbCfg.update > 0) { - printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + printf(" update: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.update); } if (g_Dbs.db[i].dbCfg.minRows > 0) { - printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + printf(" minRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.minRows); } if (g_Dbs.db[i].dbCfg.maxRows > 0) { - printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + printf(" maxRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.maxRows); } if (g_Dbs.db[i].dbCfg.comp > 0) { printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); } if (g_Dbs.db[i].dbCfg.walLevel > 0) { - printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + printf(" walLevel: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.walLevel); } if (g_Dbs.db[i].dbCfg.fsync > 0) { - printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + printf(" fsync: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.fsync); } if (g_Dbs.db[i].dbCfg.quorum > 0) { - printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + printf(" quorum: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) @@ -1586,21 +1614,26 @@ static void printfInsertMetaToFile(FILE* fp) { if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision: %s\n", + g_Dbs.db[i].dbCfg.precision); } else { - fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision error: %s\n", + g_Dbs.db[i].dbCfg.precision); } } - fprintf(fp, " super table count: %"PRIu64"\n", g_Dbs.db[i].superTblCount); - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - fprintf(fp, " super table[%"PRIu64"]:\n", j); + fprintf(fp, " super table count: %"PRIu64"\n", + g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%d]:\n", j); - fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + fprintf(fp, " stbName: %s\n", + g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + } else if (AUTO_CREATE_SUBTBL + == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "yes"); } else { fprintf(fp, " autoCreateTable: %s\n", "error"); @@ -1608,7 +1641,8 @@ static void printfInsertMetaToFile(FILE* fp) { if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + } else if (TBL_ALREADY_EXISTS + == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "yes"); } else { fprintf(fp, " childTblExists: %s\n", "error"); @@ -1640,8 +1674,10 @@ static void printfInsertMetaToFile(FILE* fp) { */ fprintf(fp, " interlaceRows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); - fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " disorderRange: %d\n", + g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", + g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].maxSqlLen); @@ -1649,23 +1685,29 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].timeStampStep); fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); - fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + fprintf(fp, " sampleFormat: %s\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", + g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", + g_Dbs.db[i].superTbls[j].tagsFile); - fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + fprintf(fp, " columnCount: %d\n ", + g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); if ((0 == strncasecmp( g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", strlen("binary"))) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + || (0 == strncasecmp( + g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", strlen("nchar")))) { fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { - fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + fprintf(fp, "column[%d]:%s ", + k, g_Dbs.db[i].superTbls[j].columns[k].dataType); } } fprintf(fp, "\n"); @@ -1678,7 +1720,8 @@ static void printfInsertMetaToFile(FILE* fp) { "binary", strlen("binary"))) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", strlen("nchar")))) { - fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, + fprintf(fp, "tag[%d]:%s(%d) ", + k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); } else { fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); @@ -3456,10 +3499,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3493,6 +3533,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; @@ -3516,9 +3561,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } @@ -3987,10 +4032,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!stbInterlaceRows) { @@ -4186,7 +4228,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { if (specifiedQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + errorPrint( + "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", __func__, __LINE__, specifiedQueryTimes->valueint); goto PARSE_OVER; @@ -4203,7 +4246,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + errorPrint( + "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); @@ -4242,15 +4286,15 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 0; + g_queryInfo.specifiedQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); @@ -4295,7 +4339,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } - tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], + sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); cJSON* resubAfterConsume = cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); @@ -4310,10 +4355,13 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { - tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + if ((NULL != result) && (result->type == cJSON_String) + && (result->valuestring != NULL)) { + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { - memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + memset(g_queryInfo.specifiedQueryInfo.result[j], + 0, MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; @@ -4420,27 +4468,27 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 0; + g_queryInfo.superQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } - cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (subkeepProgress && - subkeepProgress->type == cJSON_String - && subkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", subkeepProgress->valuestring)) { + cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (superkeepProgress && + superkeepProgress->type == cJSON_String + && superkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + } else if (0 == strcmp("no", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { - printf("ERROR: failed to read json, subscribe keepProgress error\n"); + printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); goto PARSE_OVER; } } else { @@ -4637,16 +4685,22 @@ static int getRowDataFromSample( return dataLen; } -static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) { +static int64_t generateStbRowData( + SSuperTable* stbInfo, + char* recBuf, int64_t timestamp + ) { int64_t dataLen = 0; char *pstr = recBuf; int64_t maxLen = MAX_DATA_SIZE; - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "(%" PRId64 ",", timestamp); for (int i = 0; i < stbInfo->columnCount; i++) { - if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY"))) - || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, + "BINARY", strlen("BINARY"))) + || (0 == strncasecmp(stbInfo->columns[i].dataType, + "NCHAR", strlen("NCHAR")))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { errorPrint( "binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); @@ -4708,7 +4762,7 @@ static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stb } static int64_t generateData(char *recBuf, char **data_type, - int num_of_cols, int64_t timestamp, int lenOfBinary) { + int64_t timestamp, int lenOfBinary) { memset(recBuf, 0, MAX_DATA_SIZE); char *pstr = recBuf; pstr += sprintf(pstr, "(%" PRId64, timestamp); @@ -4859,99 +4913,52 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t table } } -static int64_t generateDataTail( - SSuperTable* superTblInfo, - uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { +static int64_t generateDataTailWithoutStb( + uint64_t batch, char* buffer, + int64_t remainderBufLen, int64_t insertRows, + uint64_t startFrom, int64_t startTime, + /* int64_t *pSamplePos, */int64_t *dataLen) { + uint64_t len = 0; - uint32_t ncols_per_record = 1; // count first col ts - char *pstr = buffer; - if (superTblInfo == NULL) { - uint32_t datatypeSeq = 0; - while(g_args.datatype[datatypeSeq]) { - datatypeSeq ++; - ncols_per_record ++; - } - } - verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); - uint64_t k = 0; + int64_t k = 0; for (k = 0; k < batch;) { char data[MAX_DATA_SIZE]; memset(data, 0, MAX_DATA_SIZE); int64_t retLen = 0; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample"))) { - retLen = getRowDataFromSample( - data, - remainderBufLen, - startTime + superTblInfo->timeStampStep * k, - superTblInfo, - pSamplePos); - } else if (0 == strncasecmp(superTblInfo->dataSource, - "rand", strlen("rand"))) { + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; - int64_t randTail = superTblInfo->timeStampStep * k; - if (superTblInfo->disorderRatio > 0) { - int rand_num = taosRandom() % 100; - if(rand_num < superTblInfo->disorderRatio) { - randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); - debugPrint("rand data generated, back %"PRId64"\n", randTail); - } - } + int64_t randTail = DEFAULT_TIMESTAMP_STEP * k; - int64_t d = startTime - + randTail; - retLen = generateRowData( - data, - d, - superTblInfo); - } - - if (retLen > remainderBufLen) { - break; - } - - pstr += snprintf(pstr , retLen + 1, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - } else { - char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; - - int64_t randTail = DEFAULT_TIMESTAMP_STEP * k; - - if (g_args.disorderRatio != 0) { + if (g_args.disorderRatio != 0) { int rand_num = taosRandom() % 100; if (rand_num < g_args.disorderRatio) { - randTail = (randTail + (taosRandom() % g_args.disorderRange + 1)) * (-1); + randTail = (randTail + + (taosRandom() % g_args.disorderRange + 1)) * (-1); debugPrint("rand data generated, back %"PRId64"\n", randTail); } - } else { + } else { randTail = DEFAULT_TIMESTAMP_STEP * k; - } + } - retLen = generateData(data, data_type, - ncols_per_record, - startTime + randTail, - lenOfBinary); + retLen = generateData(data, data_type, + startTime + randTail, + lenOfBinary); - if (len > remainderBufLen) + if (len > remainderBufLen) break; - pstr += sprintf(pstr, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - } + pstr += sprintf(pstr, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); @@ -4967,17 +4974,107 @@ static int64_t generateDataTail( return k; } -static int generateSQLHead(char *tableName, int32_t tableSeq, - threadInfo* pThreadInfo, SSuperTable* superTblInfo, +static int64_t generateStbDataTail( + SSuperTable* superTblInfo, + uint64_t batch, char* buffer, + int64_t remainderBufLen, int64_t insertRows, + uint64_t startFrom, int64_t startTime, + int64_t *pSamplePos, int64_t *dataLen) { + uint64_t len = 0; + + char *pstr = buffer; + + verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); + + int64_t k = 0; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + int64_t retLen = 0; + + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample"))) { + retLen = getRowDataFromSample( + data, + remainderBufLen, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } else if (0 == strncasecmp(superTblInfo->dataSource, + "rand", strlen("rand"))) { + int64_t randTail = superTblInfo->timeStampStep * k; + if (superTblInfo->disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if(rand_num < superTblInfo->disorderRatio) { + randTail = (randTail + + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %"PRId64"\n", randTail); + } + } + + int64_t d = startTime + randTail; + retLen = generateStbRowData(superTblInfo, data, d); + } + + if (retLen > remainderBufLen) { + break; + } + + pstr += snprintf(pstr , retLen + 1, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + startFrom ++; + + if (startFrom >= insertRows) { + break; + } + } + + *dataLen = len; + return k; +} + + +static int generateSQLHeadWithoutStb(char *tableName, + char *dbName, char *buffer, int remainderBufLen) { int len; -#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. char headBuf[HEAD_BUFF_LEN]; - if (superTblInfo) { - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s values", + dbName, + tableName); + + if (len > remainderBufLen) + return -1; + + tstrncpy(buffer, headBuf, len + 1); + + return len; +} + +static int generateStbSQLHead( + SSuperTable* superTblInfo, + char *tableName, int32_t tableSeq, + char *dbName, + char *buffer, int remainderBufLen) +{ + int len; + + char headBuf[HEAD_BUFF_LEN]; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); @@ -4996,9 +5093,9 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, headBuf, HEAD_BUFF_LEN, "%s.%s using %s.%s tags %s values", - pThreadInfo->db_name, + dbName, tableName, - pThreadInfo->db_name, + dbName, superTblInfo->sTblName, tagsValBuf); tmfree(tagsValBuf); @@ -5007,22 +5104,14 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, headBuf, HEAD_BUFF_LEN, "%s.%s values", - pThreadInfo->db_name, + dbName, tableName); } else { len = snprintf( headBuf, HEAD_BUFF_LEN, "%s.%s values", - pThreadInfo->db_name, - tableName); - } - } else { - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s values", - pThreadInfo->db_name, + dbName, tableName); } @@ -5034,7 +5123,8 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, return len; } -static int64_t generateInterlaceDataBuffer( +static int64_t generateStbInterlaceData( + SSuperTable *superTblInfo, char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, @@ -5044,10 +5134,11 @@ static int64_t generateInterlaceDataBuffer( { assert(buffer); char *pstr = buffer; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, - superTblInfo, pstr, *pRemainderBufLen); + int headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, + pstr, *pRemainderBufLen); if (headLen <= 0) { return 0; @@ -5065,19 +5156,15 @@ static int64_t generateInterlaceDataBuffer( pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { startTime = taosGetTimestamp(pThreadInfo->time_precision); - } - } else { - startTime = 1500000000000; } - int64_t k = generateDataTail( - superTblInfo, - batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, - startTime, - &(pThreadInfo->samplePos), &dataLen); + int64_t k = generateStbDataTail( + superTblInfo, + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &(pThreadInfo->samplePos), &dataLen); if (k == batchPerTbl) { pstr += dataLen; @@ -5093,34 +5180,66 @@ static int64_t generateInterlaceDataBuffer( return k; } -static int64_t generateProgressiveDataBuffer( +static int64_t generateInterlaceDataWithoutStb( + char *tableName, uint64_t batchPerTbl, + uint64_t tableSeq, + char *dbName, char *buffer, + int64_t insertRows, + uint64_t *pRemainderBufLen) +{ + assert(buffer); + char *pstr = buffer; + + int headLen = generateSQLHeadWithoutStb( + tableName, dbName, + pstr, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen = 0; + + int64_t startTime = 1500000000000; + int64_t k = generateDataTailWithoutStb( + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &dataLen); + + if (k == batchPerTbl) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n", + __func__, __LINE__, k, batchPerTbl); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } + + return k; +} + +static int64_t generateStbProgressiveData( + SSuperTable *superTblInfo, char *tableName, int64_t tableSeq, - threadInfo *pThreadInfo, char *buffer, + char *dbName, char *buffer, int64_t insertRows, uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - - int ncols_per_record = 1; // count first col ts - - if (superTblInfo == NULL) { - int datatypeSeq = 0; - while(g_args.datatype[datatypeSeq]) { - datatypeSeq ++; - ncols_per_record ++; - } - } - assert(buffer != NULL); char *pstr = buffer; - int64_t k = 0; - memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo, + int64_t headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, dbName, buffer, *pRemainderBufLen); if (headLen <= 0) { @@ -5130,12 +5249,43 @@ static int64_t generateProgressiveDataBuffer( *pRemainderBufLen -= headLen; int64_t dataLen; - k = generateDataTail(superTblInfo, - g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, + + return generateStbDataTail(superTblInfo, + g_args.num_of_RPR, pstr, *pRemainderBufLen, + insertRows, startFrom, startTime, pSamplePos, &dataLen); +} - return k; +static int64_t generateProgressiveDataWithoutStb( + char *tableName, + int64_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, + int64_t *pRemainderBufLen) +{ + assert(buffer != NULL); + char *pstr = buffer; + + memset(buffer, 0, *pRemainderBufLen); + + int64_t headLen = generateSQLHeadWithoutStb( + tableName, pThreadInfo->db_name, + buffer, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen; + + return generateDataTailWithoutStb( + g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, + startTime, + /*pSamplePos, */&dataLen); } static void printStatPerThread(threadInfo *pThreadInfo) @@ -5147,12 +5297,16 @@ static void printStatPerThread(threadInfo *pThreadInfo) (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0))); } +// sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); int64_t insertRows; uint64_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5165,57 +5319,28 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } else { interlaceRows = superTblInfo->interlaceRows; } + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; } else { insertRows = g_args.num_of_DPT; interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); + if (interlaceRows > insertRows) interlaceRows = insertRows; if (interlaceRows > g_args.num_of_RPR) interlaceRows = g_args.num_of_RPR; - int progOrInterlace; - - if (interlaceRows > 0) { - progOrInterlace= INTERLACE_INSERT_MODE; - } else { - progOrInterlace = PROGRESSIVE_INSERT_MODE; - } - - uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - char tableName[TSDB_TABLE_NAME_LEN]; - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - - uint64_t insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, - pThreadInfo->ntables, insertRows); - - int64_t startTime = pThreadInfo->start_time; - uint64_t batchPerTbl = interlaceRows; uint64_t batchPerTblTimes; @@ -5226,13 +5351,30 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { batchPerTblTimes = 1; } + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + uint64_t generatedRecPerTbl = 0; bool flagSleep = true; uint64_t sleepTimeTotal = 0; - char *strInsertInto = "insert into "; - int nInsertBufLen = strlen(strInsertInto); - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampMs(); @@ -5244,13 +5386,16 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { char *pstr = pThreadInfo->buffer; - int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; uint64_t recOfBatch = 0; for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; + getTableName(tableName, pThreadInfo, tableSeq); if (0 == strlen(tableName)) { errorPrint("[%d] %s() LN%d, getTableName return null\n", @@ -5260,13 +5405,25 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } uint64_t oldRemainderLen = remainderBufLen; - int64_t generated = generateInterlaceDataBuffer( - tableName, batchPerTbl, i, batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); + + int64_t generated; + if (superTblInfo) { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + &remainderBufLen); + } debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", pThreadInfo->threadID, __func__, __LINE__, generated); @@ -5287,8 +5444,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - if (progOrInterlace == INTERLACE_INSERT_MODE) { - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table tableSeq = pThreadInfo->start_table_from; generatedRecPerTbl += batchPerTbl; @@ -5306,7 +5462,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) break; - } } verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", @@ -5384,19 +5539,18 @@ free_of_interlace: return NULL; } -// sync insertion -/* - 1 thread: 100 tables * 2000 rows/s - 1 thread: 10 tables * 20000 rows/s - 6 thread: 300 tables * 2000 rows/s - - 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s -*/ +// sync insertion progressive data static void* syncWriteProgressive(threadInfo *pThreadInfo) { debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); SSuperTable* superTblInfo = pThreadInfo->superTblInfo; uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int64_t timeStampStep = + superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t insertRows = + (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", + __func__, __LINE__, insertRows); pThreadInfo->buffer = calloc(maxSqlLen, 1); if (NULL == pThreadInfo->buffer) { @@ -5410,8 +5564,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { uint64_t startTs = taosGetTimestampMs(); uint64_t endTs; - int64_t timeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; /* int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; uint64_t st = 0; @@ -5423,21 +5575,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->samplePos = 0; - for (uint64_t tableSeq = - pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; - tableSeq ++) { + for (uint64_t tableSeq = pThreadInfo->start_table_from; + tableSeq <= pThreadInfo->end_table_to; + tableSeq ++) { int64_t start_time = pThreadInfo->start_time; - int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; - verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); - for (uint64_t i = 0; i < insertRows;) { - /* - if (insert_interval) { - st = taosGetTimestampMs(); - } - */ - char tableName[TSDB_TABLE_NAME_LEN]; getTableName(tableName, pThreadInfo, tableSeq); verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", @@ -5446,18 +5589,28 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; - int nInsertBufLen = strlen("insert into "); - int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into "); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; - int64_t generated = generateProgressiveDataBuffer( + int64_t generated; + if (superTblInfo) { + generated = generateStbProgressiveData( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows, + i, start_time, + &(pThreadInfo->samplePos), + &remainderBufLen); + } else { + generated = generateProgressiveDataWithoutStb( tableName, tableSeq, pThreadInfo, pstr, insertRows, i, start_time, &(pThreadInfo->samplePos), &remainderBufLen); + } if (generated > 0) i += generated; else @@ -5502,27 +5655,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { if (i >= insertRows) break; -/* - if (insert_interval) { - et = taosGetTimestampMs(); - - if (insert_interval > ((et - st)) ) { - int sleep_time = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %d ms for insert interval\n", - __func__, __LINE__, sleep_time); - taosMsleep(sleep_time); // ms - } - } - */ } // num_of_DPT - if (g_args.verbose_print) { - if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && + if ((g_args.verbose_print) && + (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && (0 == strncasecmp( superTblInfo->dataSource, "sample", strlen("sample")))) { verbosePrint("%s() LN%d samplePos=%"PRId64"\n", __func__, __LINE__, pThreadInfo->samplePos); - } } } // tableSeq @@ -5557,7 +5697,6 @@ static void* syncWrite(void *sarg) { // progressive mode return syncWriteProgressive(pThreadInfo); } - } static void callBack(void *param, TAOS_RES *res, int code) { @@ -5595,10 +5734,12 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != pThreadInfo->superTblInfo->disorderRatio && rand_num < pThreadInfo->superTblInfo->disorderRatio) { - int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); - generateRowData(data, d, pThreadInfo->superTblInfo); + int64_t d = pThreadInfo->lastTs + - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + generateStbRowData(pThreadInfo->superTblInfo, data, d); } else { - generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo); + generateStbRowData(pThreadInfo->superTblInfo, + data, pThreadInfo->lastTs += 1000); } pstr += sprintf(pstr, "%s", data); pThreadInfo->counter++; @@ -5769,19 +5910,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) > superTblInfo->childTblCount)) { printf("WARNING: specified offset + limit > child table count!\n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && (0 == superTblInfo->childTblLimit)) { printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } superTblInfo->childTblName = (char*)calloc(1, @@ -6129,6 +6264,13 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } +} static int insertTestProcess() { @@ -6149,10 +6291,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6422,10 +6561,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6712,13 +6848,18 @@ static void *superSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[i]); if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); appendResultToFile(res, pThreadInfo->fp); } consumed[i] ++; @@ -6813,9 +6954,15 @@ static void *specifiedSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms res = taos_consume(tsub); if (res) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + consumed ++; if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) && (consumed >= @@ -6852,10 +6999,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -7113,17 +7257,21 @@ static void setParaFromArg(){ if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + for (int i = g_Dbs.db[0].superTbls[0].columnCount; + i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } - tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, + "BINARY", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -7343,6 +7491,9 @@ int main(int argc, char *argv[]) { } else { testCmdLine(); } + + if (g_dupstr) + free(g_dupstr); } return 0; diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 595dcca413..522070e909 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -35,6 +35,8 @@ void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser); void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); +int32_t mnodeCompactAccts(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h index a5af544dc2..db258ae6d6 100644 --- a/src/mnode/inc/mnodeCluster.h +++ b/src/mnode/inc/mnodeCluster.h @@ -25,6 +25,8 @@ void mnodeCleanupCluster(); void mnodeUpdateClusterId(); const char* mnodeGetClusterId(); +int32_t mnodeCompactCluster(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index da0865833d..0fa1a15e2d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -41,6 +41,8 @@ void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); void mnodeDropAllDbs(SAcctObj *pAcct); +int32_t mnodeCompactDbs(); + // util func void mnodeAddSuperTableIntoDb(SDbObj *pDb); void mnodeRemoveSuperTableFromDb(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..2db794a173 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,6 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); +int32_t mnodeCompactDnodes(); extern int32_t tsAccessSquence; #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index ffdec02eb6..66e9eb1e0e 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -50,6 +50,7 @@ char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); +int32_t mnodeCompactMnodes(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index e4e4a7a054..2ae0c47902 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,6 +92,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbInsertCompactRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); @@ -106,6 +107,7 @@ int32_t sdbGetId(void *pTable); uint64_t sdbGetVersion(); bool sdbCheckRowDeleted(void *pTable, void *pRow); +int32_t mnodeCompactWal(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index bf04f26a90..c583a60c7a 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -36,6 +36,7 @@ void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); +int32_t mnodeCompactTables(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 156bc7aaeb..b8f0805120 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -33,6 +33,8 @@ char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); +int32_t mnodeCompactUsers(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7b798c23f8..73b0e6ae1b 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -32,6 +32,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +int32_t mnodeCompactVgroups(); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeCancelGetNextVgroup(void *pIter); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index afe474df6b..64cfa28917 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -238,6 +238,32 @@ static int32_t mnodeCreateRootAcct() { return sdbInsertRow(&row); } +int32_t mnodeCompactAccts() { + void *pIter = NULL; + SAcctObj *pAcct = NULL; + + mInfo("start to compact accts table..."); + + while (1) { + pIter = mnodeGetNextAcct(pIter, &pAcct); + if (pAcct == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, + }; + + mInfo("compact accts %s", pAcct->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact accts table..."); + + return 0; +} + #ifndef _ACCT int32_t acctInit() { return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 169d2ebd9d..553e8446ab 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -237,3 +237,27 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, pShow->numOfReads += numOfRows; return numOfRows; } + +int32_t mnodeCompactCluster() { + SClusterObj *pCluster = NULL; + void *pIter; + + mInfo("start to compact cluster table..."); + + pIter = mnodeGetNextCluster(NULL, &pCluster); + while (pCluster) { + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, + }; + + sdbInsertCompactRow(&row); + + pIter = mnodeGetNextCluster(pIter, &pCluster); + } + + mInfo("end to compact cluster table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..5e06faaad9 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; @@ -1271,3 +1271,30 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } + +int32_t mnodeCompactDbs() { + void *pIter = NULL; + SDbObj *pDb = NULL; + + mInfo("start to compact dbs table..."); + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + }; + + mInfo("compact dbs %s", pDb->name); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dbs table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 3525bcac18..51f16e4bc6 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -1270,3 +1270,30 @@ char* dnodeRoles[] = { "vnode", "any" }; + +int32_t mnodeCompactDnodes() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + + mInfo("start to compact dnodes table..."); + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .rowSize = sizeof(SDnodeObj), + }; + + mInfo("compact dnode %d", pDnode->dnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 7ef0488c42..d3511a4e62 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,6 +57,18 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; +static SStep tsMnodeCompactSteps[] = { + {"cluster", mnodeCompactCluster, NULL}, + {"dnodes", mnodeCompactDnodes, NULL}, + {"mnodes", mnodeCompactMnodes, NULL}, + {"accts", mnodeCompactAccts, NULL}, + {"users", mnodeCompactUsers, NULL}, + {"dbs", mnodeCompactDbs, NULL}, + {"vgroups", mnodeCompactVgroups, NULL}, + {"tables", mnodeCompactTables, NULL}, + +}; + static void mnodeInitTimer(); static void mnodeCleanupTimer(); static bool mnodeNeedStart() ; @@ -71,6 +83,11 @@ static int32_t mnodeInitComponents() { return dnodeStepInit(tsMnodeSteps, stepSize); } +int32_t mnodeCompactComponents() { + int32_t stepSize = sizeof(tsMnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeCompactSteps, stepSize); +} + int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ca6d6400ae..ddc9ea59c4 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -566,3 +566,30 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } + +int32_t mnodeCompactMnodes() { + void *pIter = NULL; + SMnodeObj *pMnode = NULL; + + mInfo("start to compact mnodes table..."); + + while (1) { + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .rowSize = sizeof(SMnodeObj), + }; + + mInfo("compact mnode %d", pMnode->mnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact mnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 505d3c519c..9e3b87671e 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -20,6 +20,7 @@ #include "tutil.h" #include "tref.h" #include "tbn.h" +#include "tfs.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" @@ -450,6 +451,12 @@ int32_t sdbInit() { } tsSdbMgmt.status = SDB_STATUS_SERVING; + + if (tsCompactMnodeWal) { + mnodeCompactWal(); + exit(EXIT_SUCCESS); + } + return TSDB_CODE_SUCCESS; } @@ -726,6 +733,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * } } +int32_t sdbInsertCompactRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); +} + int32_t sdbInsertRow(SSdbRow *pRow) { SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -1138,3 +1151,46 @@ static void *sdbWorkerFp(void *pWorker) { int32_t sdbGetReplicaNum() { return tsSdbMgmt.cfg.replica; } + +int32_t mnodeCompactWal() { + sdbInfo("vgId:1, start compact mnode wal..."); + + // close old wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // reset version,then compacted wal log can start from version 1 + tsSdbMgmt.version = 0; + + // change wal to wal_tmp dir + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; + char temp[TSDB_FILENAME_LEN] = {0}; + sprintf(temp, "%s/wal", tsMnodeTmpDir); + tsSdbMgmt.wal = walOpen(temp, &walCfg); + walRenew(tsSdbMgmt.wal); + + // compact memory tables info to wal tmp dir + if (mnodeCompactComponents() != 0) { + tfsRmdir(tsMnodeTmpDir); + return -1; + } + + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // rename old wal to wal_bak + if (taosRename(tsMnodeDir, tsMnodeBakDir) != 0) { + return -1; + } + + // rename wal_tmp to wal + if (taosRename(tsMnodeTmpDir, tsMnodeDir) != 0) { + return -1; + } + + // del wal_tmp dir + sdbInfo("vgId:1, compact mnode wal success"); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2a8e941fcb..4d3125a2d1 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3242,3 +3242,65 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro return numOfRows; } + +static int32_t mnodeCompactSuperTables() { + void *pIter = NULL; + SSTableObj *pTable = NULL; + + mInfo("start to compact super table..."); + + while (1) { + pIter = mnodeGetNextSuperTable(pIter, &pTable); + if (pTable == NULL) break; + + int32_t schemaSize = (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, + .rowSize = sizeof(SSTableObj) + schemaSize, + }; + + mInfo("compact super %" PRIu64, pTable->uid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact super table..."); + + return 0; +} + +static int32_t mnodeCompactChildTables() { + void *pIter = NULL; + SCTableObj *pTable = NULL; + + mInfo("start to compact child table..."); + + while (1) { + pIter = mnodeGetNextChildTable(pIter, &pTable); + if (pTable == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + }; + + mInfo("compact child %" PRIu64 ":%d", pTable->uid, pTable->tid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact child table..."); + + return 0; +} + +int32_t mnodeCompactTables() { + mnodeCompactSuperTables(); + + mnodeCompactChildTables(); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index e77c1b3e59..c5c54791cf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -617,3 +617,30 @@ static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } + +int32_t mnodeCompactUsers() { + void *pIter = NULL; + SUserObj *pUser = NULL; + + mInfo("start to compact users table..."); + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + }; + + mInfo("compact users %s", pUser->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact users table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7222c8d1a0..67532ad85a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -1302,3 +1302,30 @@ void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); cver[2] = (int8_t)(iver % 100); } + +int32_t mnodeCompactVgroups() { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + + mInfo("start to compact vgroups table..."); + + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + }; + + mInfo("compact vgroups %d", pVgroup->vgId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact vgroups table..."); + + return 0; +} \ No newline at end of file diff --git a/src/os/inc/osMips64.h b/src/os/inc/osMips64.h new file mode 100644 index 0000000000..ed7b08a311 --- /dev/null +++ b/src/os/inc/osMips64.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_MIPS64_H +#define TDENGINE_OS_MIPS64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c index 4467a607b2..33cc39e112 100644 --- a/src/os/src/detail/osSignal.c +++ b/src/os/src/detail/osSignal.c @@ -25,14 +25,14 @@ typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp) { - struct sigaction act = {{0}}; + struct sigaction act; memset(&act, 0, sizeof(act)); #if 1 act.sa_flags = SA_SIGINFO; act.sa_sigaction = (FLinuxSignalHandler)sigfp; -#else - act.sa_handler = sigfp; -#endif - sigaction(signum, &act, NULL); +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); } void taosIgnSignal(int32_t signum) { diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 94af8e3ecd..d9f7d81ebd 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -417,3 +417,13 @@ void monExecuteSQL(char *sql) { monDebug("execute sql:%s", sql); taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); } + +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) { + if (tsMonitor.conn == NULL) { + callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID); + return; + } + + monDebug("execute sql:%s", sql); + taos_query_a(tsMonitor.conn, sql, callback, param); +} diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 7742d151a0..3b5c2b4cfb 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -40,7 +40,7 @@ typedef struct SHeapEntry { } SHeapEntry; typedef struct SHistogramInfo { - int32_t numOfElems; + int64_t numOfElems; int32_t numOfEntries; int32_t maxEntries; double min; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index ae25a75234..5fa35d0ee5 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -446,7 +446,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { } void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %d\n", pHisto->numOfEntries, pHisto->numOfElems); + printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); #if defined(USE_ARRAYLIST) for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c index 4009973a9f..d2b63eb9ee 100644 --- a/src/util/src/tcrc32c.c +++ b/src/util/src/tcrc32c.c @@ -18,7 +18,7 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef _TD_ARM_ +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index ca3746c598..f133101cca 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,12 +120,14 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } + if (pVnode->dropped) { + vnodeRelease(pVnode); + return TSDB_CODE_SUCCESS; + } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; - // remove from hash, so new messages wont be consumed - vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); @@ -425,6 +427,10 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; + if (pVnode->dropped) { + vnodeRelease(pVnode); + return 0; + } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); @@ -510,6 +516,8 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); + vnodeRemoveFromHash(pVnode); + // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index b28bdbf130..2831090267 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,14 +117,17 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + if (pVnode->dropped) { + return TSDB_CODE_APP_NOT_READY; + } + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } - SVnodeObj *pVnode = vparam; - int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index c482d1fd1a..1eaddc3d25 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,6 +66,9 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + if (pVnode->status == TAOS_VN_STATUS_CLOSING) + return true; + while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 05af34a34f..e5a1964915 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,6 +55,11 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while notify role", vgId); + vnodeRelease(pVnode); + return; + } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -75,6 +80,11 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); + vnodeRelease(pVnode); + return; + } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -129,6 +139,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); + vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 36516d81df..16089c8e91 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -340,8 +340,11 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { if (pWrite->processedCount >= 100) { vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), pWrite->processedCount); - pWrite->processedCount = 1; - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + void *handle = pWrite->rpcMsg.handle; + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + SRpcMsg rpcRsp = {.handle = handle, .code = code}; + rpcSendResponse(&rpcRsp); } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { @@ -386,4 +389,6 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } + + taosMsleep(900); } diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 9a16084683..c6d4687ed7 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -74,7 +74,7 @@ function runQueryPerfTest { python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT + #python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT } diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index b50180e2b3..a32cba167e 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -104,7 +104,7 @@ class taosdemoPerformace: return output def insertData(self): - os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index dfe521b92b..7c4c883fb1 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum, actual: $data00 + print expect $rowNum , actual: $data00 return -1 endi $i = $i + 1