From c815a1b20d3c908b8f1a1d4430a467a281f51b1b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 11 May 2020 21:25:48 +0800 Subject: [PATCH 1/8] add tag and alter-table cases. [TD-280] --- tests/pytest/alter/__init__.py | 0 tests/pytest/alter/alter_replica.py | 129 ++++++++++++++++++++++++++ tests/pytest/alter/alter_stable.py | 138 ++++++++++++++++++++++++++++ tests/pytest/alter/alter_table.py | 138 ++++++++++++++++++++++++++++ tests/pytest/alter/file_corrupt.py | 77 ++++++++++++++++ tests/pytest/tag_lite/__init__.py | 0 tests/pytest/tag_lite/datatype.py | 135 +++++++++++++++++++++++++++ 7 files changed, 617 insertions(+) create mode 100644 tests/pytest/alter/__init__.py create mode 100644 tests/pytest/alter/alter_replica.py create mode 100644 tests/pytest/alter/alter_stable.py create mode 100644 tests/pytest/alter/alter_table.py create mode 100644 tests/pytest/alter/file_corrupt.py create mode 100644 tests/pytest/tag_lite/__init__.py create mode 100644 tests/pytest/tag_lite/datatype.py diff --git a/tests/pytest/alter/__init__.py b/tests/pytest/alter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/alter/alter_replica.py b/tests/pytest/alter/alter_replica.py new file mode 100644 index 0000000000..6cf0f65825 --- /dev/null +++ b/tests/pytest/alter/alter_replica.py @@ -0,0 +1,129 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self): + tdLog.debug("start to execute %s" % __file__) + tdLog.info("prepare cluster") + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + + self.conn = taos.connect(config=tdDnodes.getSimCfgPath()) + tdSql.init(self.conn.cursor()) + tdSql.execute('reset query cache') + tdSql.execute('create dnode 192.168.0.2') + tdDnodes.deploy(2) + tdDnodes.start(2) + + self.conn = taos.connect(config=tdDnodes.getSimCfgPath()) + tdSql.init(self.conn.cursor()) + tdSql.execute('reset query cache') + tdSql.execute('create dnode 192.168.0.3') + tdDnodes.deploy(3) + tdDnodes.start(3) + + def run(self): + tdSql.execute('create database db replica 3 days 7') + tdSql.execute('use db') + for tid in range(1, 11): + tdSql.execute('create table tb%d(ts timestamp, i int)' % tid) + tdLog.sleep(10) + + tdLog.info("================= step1") + startTime = 1520000010000 + for rid in range(1, 11): + for tid in range(1, 11): + tdSql.execute( + 'insert into tb%d values(%ld, %d)' % + (tid, startTime, rid)) + startTime += 1 + tdSql.query('select * from tb1') + tdSql.checkRows(10) + tdLog.sleep(5) + + tdLog.info("================= step2") + tdSql.execute('alter database db replica 2') + tdLog.sleep(10) + + tdLog.info("================= step3") + for rid in range(1, 11): + for tid in range(1, 11): + tdSql.execute( + 'insert into tb%d values(%ld, %d)' % + (tid, startTime, rid)) + startTime += 1 + tdSql.query('select * from tb1') + tdSql.checkRows(20) + tdLog.sleep(5) + + tdLog.info("================= step4") + tdSql.execute('alter database db replica 1') + tdLog.sleep(10) + + tdLog.info("================= step5") + for rid in range(1, 11): + for tid in range(1, 11): + tdSql.execute( + 'insert into tb%d values(%ld, %d)' % + (tid, startTime, rid)) + startTime += 1 + tdSql.query('select * from tb1') + tdSql.checkRows(30) + tdLog.sleep(5) + + tdLog.info("================= step6") + tdSql.execute('alter database db replica 2') + tdLog.sleep(10) + + tdLog.info("================= step7") + for rid in range(1, 11): + for tid in range(1, 11): + tdSql.execute( + 'insert into tb%d values(%ld, %d)' % + (tid, startTime, rid)) + startTime += 1 + tdSql.query('select * from tb1') + tdSql.checkRows(40) + tdLog.sleep(5) + + tdLog.info("================= step8") + tdSql.execute('alter database db replica 3') + tdLog.sleep(10) + + tdLog.info("================= step9") + for rid in range(1, 11): + for tid in range(1, 11): + tdSql.execute( + 'insert into tb%d values(%ld, %d)' % + (tid, startTime, rid)) + startTime += 1 + tdSql.query('select * from tb1') + tdSql.checkRows(50) + tdLog.sleep(5) + + def stop(self): + tdSql.close() + self.conn.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addCluster(__file__, TDTestCase()) diff --git a/tests/pytest/alter/alter_stable.py b/tests/pytest/alter/alter_stable.py new file mode 100644 index 0000000000..5772edcf7f --- /dev/null +++ b/tests/pytest/alter/alter_stable.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.types = [ + "int", + "bigint", + "float", + "double", + "smallint", + "tinyint", + "binary(10)", + "nchar(10)", + "timestamp"] + self.rowNum = 300 + self.ts = 1537146000000 + self.step = 1000 + self.sqlHead = "select count(*), count(c1) " + self.sqlTail = " from stb" + + def addColumnAndCount(self): + for colIdx in range(len(self.types)): + tdSql.execute( + "alter table stb add column c%d %s" % + (colIdx + 2, self.types[colIdx])) + self.sqlHead = self.sqlHead + ",count(c%d) " % (colIdx + 2) + tdSql.query(self.sqlHead + self.sqlTail) + + # count non-NULL values in each column + tdSql.checkData(0, 0, self.rowNum * (colIdx + 1)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 1)) + for i in range(2, colIdx + 2): + print("check1: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 2)) + + # insert more rows + for k in range(self.rowNum): + self.ts += self.step + sql = "insert into tb values (%d, %d" % (self.ts, colIdx + 2) + for j in range(colIdx + 1): + sql += ", %d" % (colIdx + 2) + sql += ")" + tdSql.execute(sql) + + # count non-NULL values in each column + tdSql.query(self.sqlHead + self.sqlTail) + tdSql.checkData(0, 0, self.rowNum * (colIdx + 2)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 2)) + for i in range(2, colIdx + 2): + print("check2: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 3)) + + def dropColumnAndCount(self): + tdSql.query(self.sqlHead + self.sqlTail) + res = [] + for i in range(len(self.types)): + res.append(tdSql.getData(0, i + 2)) + + print(res) + + for colIdx in range(len(self.types), 0, -1): + tdSql.execute("alter table stb drop column c%d" % (colIdx + 2)) + # self.sqlHead = self.sqlHead + ",count(c%d) " %(colIdx + 2) + tdSql.query(self.sqlHead + self.sqlTail) + + # count non-NULL values in each column + tdSql.checkData(0, 0, self.rowNum * (colIdx + 1)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 1)) + for i in range(2, colIdx + 2): + print("check1: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 2)) + + # insert more rows + for k in range(self.rowNum): + self.ts += self.step + sql = "insert into tb values (%d, %d" % (self.ts, colIdx + 2) + for j in range(colIdx + 1): + sql += ", %d" % (colIdx + 2) + sql += ")" + tdSql.execute(sql) + + # count non-NULL values in each column + tdSql.query(self.sqlHead + self.sqlTail) + tdSql.checkData(0, 0, self.rowNum * (colIdx + 2)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 2)) + for i in range(2, colIdx + 2): + print("check2: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 3)) + + def run(self): + # Setup params + db = "db" + + # Create db + tdSql.execute("drop database if exists %s" % (db)) + tdSql.execute("reset query cache") + tdSql.execute("create database %s maxrows 200 maxtables 4" % (db)) + tdSql.execute("use %s" % (db)) + + # Create a table with one colunm of int type and insert 300 rows + tdLog.info("Create stb and tb") + tdSql.execute("create table stb (ts timestamp, c1 int) tags (tg1 int)") + tdSql.execute("create table tb using stb tags (0)") + tdLog.info("Insert %d rows into tb" % (self.rowNum)) + for k in range(1, self.rowNum + 1): + self.ts += self.step + tdSql.execute("insert into tb values (%d, 1)" % (self.ts)) + + # Alter tb and add a column of smallint type, then query tb to see if + # all added column are NULL + self.addColumnAndCount() + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query(self.sqlHead + self.sqlTail) + for i in range(2, len(self.types) + 2): + tdSql.checkData(0, i, self.rowNum * (len(self.types) + 2 - i)) + + self.dropColumnAndCount() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +#tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/alter/alter_table.py b/tests/pytest/alter/alter_table.py new file mode 100644 index 0000000000..6e0c591da6 --- /dev/null +++ b/tests/pytest/alter/alter_table.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.types = [ + "int", + "bigint", + "float", + "double", + "smallint", + "tinyint", + "binary(10)", + "nchar(10)", + "timestamp"] + self.rowNum = 300 + self.ts = 1537146000000 + self.step = 1000 + self.sqlHead = "select count(*), count(c1) " + self.sqlTail = " from tb" + + def addColumnAndCount(self): + for colIdx in range(len(self.types)): + tdSql.execute( + "alter table tb add column c%d %s" % + (colIdx + 2, self.types[colIdx])) + self.sqlHead = self.sqlHead + ",count(c%d) " % (colIdx + 2) + tdSql.query(self.sqlHead + self.sqlTail) + + # count non-NULL values in each column + tdSql.checkData(0, 0, self.rowNum * (colIdx + 1)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 1)) + for i in range(2, colIdx + 2): + print("check1: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 2)) + + # insert more rows + for k in range(self.rowNum): + self.ts += self.step + sql = "insert into tb values (%d, %d" % (self.ts, colIdx + 2) + for j in range(colIdx + 1): + sql += ", %d" % (colIdx + 2) + sql += ")" + tdSql.execute(sql) + + # count non-NULL values in each column + tdSql.query(self.sqlHead + self.sqlTail) + tdSql.checkData(0, 0, self.rowNum * (colIdx + 2)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 2)) + for i in range(2, colIdx + 2): + print("check2: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 3)) + + def dropColumnAndCount(self): + + tdSql.query(self.sqlHead + self.sqlTail) + res = [] + for i in range(len(self.types)): + res[i] = tdSql.getData(0, i + 2) + + print(res.join) + + for colIdx in range(len(self.types), 0, -1): + tdSql.execute("alter table tb drop column c%d" % (colIdx + 2)) + # self.sqlHead = self.sqlHead + ",count(c%d) " %(colIdx + 2) + tdSql.query(self.sqlHead + self.sqlTail) + + # count non-NULL values in each column + tdSql.checkData(0, 0, self.rowNum * (colIdx + 1)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 1)) + for i in range(2, colIdx + 2): + print("check1: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 2)) + + # insert more rows + for k in range(self.rowNum): + self.ts += self.step + sql = "insert into tb values (%d, %d" % (self.ts, colIdx + 2) + for j in range(colIdx + 1): + sql += ", %d" % (colIdx + 2) + sql += ")" + tdSql.execute(sql) + + # count non-NULL values in each column + tdSql.query(self.sqlHead + self.sqlTail) + tdSql.checkData(0, 0, self.rowNum * (colIdx + 2)) + tdSql.checkData(0, 1, self.rowNum * (colIdx + 2)) + for i in range(2, colIdx + 2): + print("check2: i=%d colIdx=%d" % (i, colIdx)) + tdSql.checkData(0, i, self.rowNum * (colIdx - i + 3)) + + def run(self): + # Setup params + db = "db" + + # Create db + tdSql.execute("drop database if exists %s" % (db)) + tdSql.execute("reset query cache") + tdSql.execute("create database %s maxrows 200 maxtables 4" % (db)) + tdSql.execute("use %s" % (db)) + + # Create a table with one colunm of int type and insert 300 rows + tdLog.info("Create table tb") + tdSql.execute("create table tb (ts timestamp, c1 int)") + tdLog.info("Insert %d rows into tb" % (self.rowNum)) + for k in range(1, self.rowNum + 1): + self.ts += self.step + tdSql.execute("insert into tb values (%d, 1)" % (self.ts)) + + # Alter tb and add a column of smallint type, then query tb to see if + # all added column are NULL + self.addColumnAndCount() + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query(self.sqlHead + self.sqlTail) + for i in range(2, len(self.types) + 2): + tdSql.checkData(0, i, self.rowNum * (len(self.types) + 2 - i)) + + self.dropColumnAndCount() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +#tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/alter/file_corrupt.py b/tests/pytest/alter/file_corrupt.py new file mode 100644 index 0000000000..51ea882270 --- /dev/null +++ b/tests/pytest/alter/file_corrupt.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + tdSql.execute( + 'create table st (ts timestamp, v1 int, v2 int, v3 int, v4 int, v5 int) tags (t int)') + + totalTables = 100 + batchSize = 500 + totalBatch = 60 + + tdLog.info( + "create %d tables, insert %d rows per table" % + (totalTables, batchSize * totalBatch)) + + for t in range(0, totalTables): + tdSql.execute('create table t%d using st tags(%d)' % (t, t)) + # 2019-06-10 00:00:00 + beginTs = 1560096000000 + interval = 10000 + for r in range(0, totalBatch): + sql = 'insert into t%d values ' % (t) + for b in range(0, batchSize): + ts = beginTs + (r * batchSize + b) * interval + sql += '(%d, 1, 2, 3, 4, 5)' % (ts) + tdSql.execute(sql) + + tdLog.info("insert data finished") + tdSql.execute('alter table st add column v6 int') + tdLog.sleep(5) + tdLog.info("alter table finished") + + tdSql.query("select count(*) from t50") + tdSql.checkData(0, 0, (int)(batchSize * totalBatch)) + + tdLog.info("insert") + tdSql.execute( + "insert into t50 values ('2019-06-13 07:59:55.000', 1, 2, 3, 4, 5, 6)") + + tdLog.info("import") + tdSql.execute( + "import into t50 values ('2019-06-13 07:59:55.000', 1, 2, 3, 4, 5, 6)") + + tdLog.info("query") + tdSql.query("select count(*) from t50") + tdSql.checkData(0, 0, batchSize * totalBatch + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/__init__.py b/tests/pytest/tag_lite/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/tag_lite/datatype.py b/tests/pytest/tag_lite/datatype.py new file mode 100644 index 0000000000..1fcf7ce19e --- /dev/null +++ b/tests/pytest/tag_lite/datatype.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdLog.info("================= step0") + tdSql.execute('reset query cache') + tdLog.info("drop database db if exits") + tdSql.execute('drop database if exists db') + tdLog.info("================= step1") + tdSql.execute('create database db maxtables 4') + tdLog.sleep(5) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 super table") + tdSql.execute('create table stb (ts timestamp, i int) \ + tags (tin int, tfl float, tbg bigint, tdo double, tbi binary(10), tbl bool)') + + tdLog.info("================= step2") + tdLog.info("create %d tables" % self.ntables) + for tid in range(1, self.ntables + 1): + tdSql.execute( + 'create table tb%d using stb tags(%d,%f,%ld,%f,\'%s\',%d)' % + (tid, + tid % + 3, + 1.2 * + tid, + self.startTime + + tid, + 1.22 * + tid, + 't' + + str(tid), + tid % + 2)) + tdLog.sleep(5) + + tdLog.info("================= step3") + tdLog.info( + "insert %d data in to each %d tables" % + (self.rowsPerTable, self.ntables)) + for rid in range(1, self.rowsPerTable + 1): + sqlcmd = ['insert into'] + for tid in range(1, self.ntables + 1): + sqlcmd.append( + 'tb%d values(%ld,%d)' % + (tid, self.startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + tdSql.query('select count(*) from stb') + tdSql.checkData(0, 0, self.rowsPerTable * self.ntables) + + tdLog.info("================= step4") + tdLog.info("drop one tag") + tdSql.execute('alter table stb drop tag tbi') + tdLog.info("insert %d data in to each %d tables" % (2, self.ntables)) + for rid in range(self.rowsPerTable + 1, self.rowsPerTable + 3): + sqlcmd = ['insert into'] + for tid in range(1, self.ntables + 1): + sqlcmd.append( + 'tb%d values(%ld,%d)' % + (tid, self.startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + self.rowsPerTable += 2 + tdSql.query('select count(*) from stb') + tdSql.checkData(0, 0, self.rowsPerTable * self.ntables) + tdSql.query('describe tb1') + tdSql.checkRows(2 + 5) + + tdLog.info("================= step5") + tdLog.info("add one tag") + tdSql.execute('alter table stb add tag tnc nchar(10)') + for tid in range(1, self.ntables + 1): + tdSql.execute('alter table tb%d set tag tnc=\"%s\"' % + (tid, str(tid * 1.2))) + tdLog.info("insert %d data in to each %d tables" % (2, self.ntables)) + for rid in range(self.rowsPerTable + 1, self.rowsPerTable + 3): + sqlcmd = ['insert into'] + for tid in range(1, self.ntables + 1): + sqlcmd.append( + 'tb%d values(%ld,%d)' % + (tid, self.startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + self.rowsPerTable += 2 + tdSql.query('select count(*) from stb') + tdSql.checkData(0, 0, self.rowsPerTable * self.ntables) + tdSql.query('describe tb1') + tdSql.checkRows(2 + 6) + + tdLog.info("================= step6") + tdLog.info("group and filter by tag1 int") + tdSql.query('select max(i) from stb where tbl=0 group by tin') + tdSql.checkRows(3) + tdSql.execute('reset query cache') + tdSql.query('select max(i) from stb where tbl=true group by tin') + tdSql.checkData(2, 0, self.rowsPerTable) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 9b792a8d31e77992daa9d4193bf6fec4a98bea1b Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 18 May 2020 18:23:16 +0800 Subject: [PATCH 2/8] fix failed cases --- src/client/src/tscParseInsert.c | 4 +++- src/client/src/tscServer.c | 20 +++++++++++--------- src/mnode/src/mgmtTable.c | 7 ++++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 7e67ff82e9..36b1ab5993 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1016,7 +1016,9 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); } - if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536 + // but TSDB_PAYLOAD_SIZE is 65380 + if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index bc717ed88c..28c5ae9ca0 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1487,15 +1487,16 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char * pMsg; int msgLen = 0; - char *tmpData = 0; - if (pSql->cmd.allocSize > 0) { - tmpData = calloc(1, pSql->cmd.allocSize); + char *tmpData = NULL; + uint32_t len = pSql->cmd.payloadLen; + if (len > 0) { + tmpData = calloc(1, len); if (NULL == tmpData) { return TSDB_CODE_CLI_OUT_OF_MEMORY; } // STagData is in binary format, strncpy is not available - memcpy(tmpData, pSql->cmd.payload, pSql->cmd.allocSize); + memcpy(tmpData, pSql->cmd.payload, len); } SSqlCmd * pCmd = &pSql->cmd; @@ -1509,9 +1510,9 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg); - if (pSql->cmd.autoCreated) { - memcpy(pInfoMsg->tags, tmpData, sizeof(STagData)); - pMsg += sizeof(STagData); + if (pSql->cmd.autoCreated && len > 0) { + memcpy(pInfoMsg->tags, tmpData, len); + pMsg += len; } pCmd->payloadLen = pMsg - (char*)pInfoMsg;; @@ -2394,7 +2395,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { tscError("%p malloc failed for payload to get table meta", pSql); free(pNew); @@ -2405,7 +2406,8 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name)); - memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); // tag information if table does not exists. + memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen); // tag information if table does not exists. + pNew->cmd.payloadLen = pSql->cmd.payloadLen; tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); pNew->fp = tscTableMetaCallBack; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 28b3c6e3dd..50f24eef9f 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -1736,7 +1736,12 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) { pCreateMsg->igExists = 1; pCreateMsg->getMeta = 1; pCreateMsg->contLen = htonl(contLen); - memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); + + contLen = sizeof(STagData); + if (contLen > pMsg->contLen - sizeof(SCMTableInfoMsg)) { + contLen = pMsg->contLen - sizeof(SCMTableInfoMsg); + } + memcpy(pCreateMsg->schema, pInfo->tags, contLen); SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); pMsg->pCont = newMsg->pCont; From 5b439a51223891b3e9a81c0239f2353042cd78f1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 18 May 2020 18:21:41 +0800 Subject: [PATCH 3/8] add tag_lite/filter.py [TD-351] --- tests/pytest/fulltest.sh | 3 + tests/pytest/random-test/random-test.py | 145 +++++++++++++ tests/pytest/smoketest.sh | 7 + tests/pytest/tag_lite/filter.py | 270 ++++++++++++++++++++++++ tests/pytest/util/sql.py | 46 ++-- tests/pytest/valgrind-test.sh | 7 + 6 files changed, 455 insertions(+), 23 deletions(-) create mode 100644 tests/pytest/random-test/random-test.py create mode 100644 tests/pytest/tag_lite/filter.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 8634a523f3..efd9f7ce52 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -19,6 +19,8 @@ python3 ./test.py $1 -f table/column_num.py python3 ./test.py $1 -f table/db_table.py python3 ./test.py $1 -f table/tablename-boundary.py +# tag +python3 ./test.py $1 -f tag_lite/filter.py python3 ./test.py $1 -f tag_lite/create-tags-boundary.py python3 ./test.py $1 -f dbmgmt/database-name-boundary.py @@ -96,3 +98,4 @@ python3 ./test.py $1 -f user/pass_len.py # table #python3 ./test.py $1 -f table/del_stable.py + diff --git a/tests/pytest/random-test/random-test.py b/tests/pytest/random-test/random-test.py new file mode 100644 index 0000000000..a3f4c00ea5 --- /dev/null +++ b/tests/pytest/random-test/random-test.py @@ -0,0 +1,145 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class Test: + def __init__(self): + self.current_tb = "" + self.last_tb = "" + self.written = 0 + + def create_table(self): + tdLog.info("create a table") + self.current_tb = "tb%d" % int(round(time.time() * 1000)) + tdLog.info("current table %s" % self.current_tb) + + if (self.current_tb == self.last_tb): + return + else: + tdSql.execute( + 'create table %s (ts timestamp, speed int)' % + self.current_tb) + self.last_tb = self.current_tb + self.written = 0 + + def insert_data(self): + tdLog.info("will insert data to table") + if (self.current_tb == ""): + tdLog.info("no table, create first") + self.create_table() + + tdLog.info("insert data to table") + insertRows = 10 + tdLog.info("insert %d rows to %s" % (insertRows, self.last_tb)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into %s values (now + %dm, %d)' % + (self.last_tb, i, i)) + self.written = self.written + 1 + + tdLog.info("insert earlier data") + tdSql.execute('insert into %s values (now - 5m , 10)' % self.last_tb) + self.written = self.written + 1 + tdSql.execute('insert into %s values (now - 6m , 10)' % self.last_tb) + self.written = self.written + 1 + tdSql.execute('insert into %s values (now - 7m , 10)' % self.last_tb) + self.written = self.written + 1 + tdSql.execute('insert into %s values (now - 8m , 10)' % self.last_tb) + self.written = self.written + 1 + + def query_data(self): + if (self.written > 0): + tdLog.info("query data from table") + tdSql.query("select * from %s" % self.last_tb) + tdSql.checkRows(self.written) + + def create_stable(self): + tdLog.info("create a super table") + + def restart_database(self): + tdLog.info("restart databae") + tdDnodes.stop(1) + tdDnodes.start(1) + tdLog.sleep(5) + + def force_restart(self): + tdLog.info("force restart database") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(5) + + def drop_table(self): + if (self.current_tb != ""): + tdLog.info("drop current tb %s" % self.current_tb) + tdSql.execute("drop table %s" % self.current_tb) + self.current_tb = "" + self.last_tb = "" + self.written = 0 + + def reset_query_cache(self): + tdLog.info("reset query cache") + tdSql.execute("reset query cache") + tdLog.sleep(1) + + def reset_database(self): + tdLog.info("reset database") + tdDnodes.forcestop(1) + tdDnodes.deploy(1) + self.current_tb = "" + self.last_tb = "" + self.written = 0 + tdDnodes.start(1) + tdSql.prepare() + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + test = Test() + + switch = { + 1: test.create_table, + 2: test.insert_data, + 3: test.query_data, + 4: test.create_stable, + 5: test.restart_database, + 6: test.force_restart, + 7: test.drop_table, + 8: test.reset_query_cache, + 9: test.reset_database, + } + + for x in range(1, 100): + r = random.randint(1, 9) + tdLog.notice("iteration %d run func %d" % (x, r)) + switch.get(r, lambda: "ERROR")() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index 6314b3f83b..5f48789d45 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -1,4 +1,5 @@ #!/bin/bash +# insert python3 ./test.py $1 -f insert/basic.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f insert/int.py @@ -24,6 +25,7 @@ python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f insert/multi.py python3 ./test.py $1 -s && sleep 1 +# table python3 ./test.py $1 -f table/column_name.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f table/column_num.py @@ -31,6 +33,7 @@ python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f table/db_table.py python3 ./test.py $1 -s && sleep 1 +# import python3 ./test.py $1 -f import_merge/importDataLastSub.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importHead.py @@ -43,3 +46,7 @@ python3 ./test.py $1 -f import_merge/importTail.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importTRestart.py python3 ./test.py $1 -s && sleep 1 + +#tag +python3 ./test.py $1 -f tag_lite/filter.py +python3 ./test.py $1 -s && sleep 1 diff --git a/tests/pytest/tag_lite/filter.py b/tests/pytest/tag_lite/filter.py new file mode 100644 index 0000000000..7d160a1b61 --- /dev/null +++ b/tests/pytest/tag_lite/filter.py @@ -0,0 +1,270 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + #TSIM: system sh/stop_dnodes.sh + #TSIM: system sh/deploy.sh -n dnode1 -i 1 + #TSIM: system sh/exec.sh -n dnode1 -s start + #TSIM: + #TSIM: sleep 3000 + #TSIM: sql connect + #TSIM: + #TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + #TSIM: + dbPrefix = "ta_fi_db" + tbPrefix = "ta_fi_tb" + mtPrefix = "ta_fi_mt" + #TSIM: $tbNum = 10 + rowNum = 20 + #TSIM: $totalNum = 200 + #TSIM: + #TSIM: print =============== step1 + tdLog.info('=============== step1') + i = 0 + #TSIM: $db = $dbPrefix . $i + mt = "%s%d" % (mtPrefix, i) + #TSIM: + #TSIM: sql create database $db + #TSIM: sql use $db + #TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol binary(10)) + tdLog.info("create table %s (ts timestamp, tbcol int) TAGS(tgcol binary(10))" % mt) + tdSql.execute('create table %s (ts timestamp, tbcol int) TAGS(tgcol binary(10))' % mt) + #TSIM: + i = 0 + while (i < 5): + tb = "tbPrefix%d" % i + tdLog.info("create table %s using %s tags( '0' )" % (tb, mt)) + tdSql.execute("create table %s using %s tags( '0' )" % (tb, mt)) + + x = 0 + while (x < rowNum): + ms = "%dm" % x + tdLog.info("insert into %s values (now + %s , %d)" % (tb, ms, x)) + tdSql.execute("insert into %s values (now + %s , %d)" % (tb, ms, x)) + x = x + 1 + i = i + 1 + + while (i < 10): + tb = "%s%d" % (tbPrefix , i) + #TSIM: sql create table $tb using $mt tags( '1' ) + tdLog.info("create table %s using %s tags( '1' )" % (tb, mt)) + tdSql.execute("create table %s using %s tags( '1' )" % (tb, mt)) + x = 0 + while (x < rowNum): + ms = "%dm" % x + #TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info("insert into %s values (now + %s, %d )" % (tb, ms, x)) + tdSql.execute("insert into %s values (now + %s, %d )" % (tb, ms, x)) + x = x + 1 + i = i + 1 + #TSIM: + #TSIM: print =============== step2 + tdLog.info('=============== step2') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = '1' + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = '1'" % mt) + tdSql.query("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = '1'" % mt) + #TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info("%s %s %s %s %s %s %s" % (tdSql.getData(0, 0), tdSql.getData(0, 1), tdSql.getData(0, 2), tdSql.getData(0, 3), tdSql.getData(0, 4), tdSql.getData(0, 5), tdSql.getData(0, 6))) + #TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tg = '1' -x step2 + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tg = '1' -x step2" % mt) + tdSql.error("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tg = '1'" % mt) + #TSIM: return -1 + #TSIM: step2: + #TSIM: + #TSIM: print =============== step3 + tdLog.info('=============== step3') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where noexist = '1' -x step3 + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where noexist = '1' -x step3" % mt) + tdSql.error("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where noexist = '1'" % mt) + #TSIM: return -1 + #TSIM: step3: + #TSIM: + #TSIM: print =============== step4 + tdLog.info('=============== step4') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tbcol = '1'" % mt) + tdSql.query("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tbcol = '1'" % mt) + #TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + #TSIM: return -1 + #TSIM: endi + #TSIM: if $data00 != 10 then + tdLog.info('tdSql.checkData(0, 0, 10)') + tdSql.checkData(0, 0, 10) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: print =============== step5 + tdLog.info('=============== step5') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s" % mt) + tdSql.query("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s" % mt) + #TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info("%s %s %s %s %s %s %s" % (tdSql.getData(0,0), tdSql.getData(0,1), tdSql.getData(0,2), tdSql.getData(0, 3), tdSql.getData(0, 4), tdSql.getData(0,5 ), tdSql.getData(0, 6))) + #TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: print =============== step6 + tdLog.info('=============== step6') + #TSIM: sql select count(tbcol), avg(cc), sum(xx), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt -x step6 + tdLog.info("select count(tbcol), avg(cc), sum(xx), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s -x step6" % mt) + tdSql.error("select count(tbcol), avg(cc), sum(xx), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s" % mt) + #TSIM: return -1 + #TSIM: step6: + #TSIM: + #TSIM: print =============== step7 + tdLog.info('=============== step7') + #TSIM: sql select count(tgcol), avg(tgcol), sum(tgcol), min(tgcol), max(tgcol), first(tgcol), last(tgcol) from $mt -x step7 + tdLog.info("select count(tgcol), avg(tgcol), sum(tgcol), min(tgcol), max(tgcol), first(tgcol), last(tgcol) from %s -x step7" % mt) + tdSql.error("select count(tgcol), avg(tgcol), sum(tgcol), min(tgcol), max(tgcol), first(tgcol), last(tgcol) from %s" % mt) + #TSIM: return -1 + #TSIM: step7: + #TSIM: + #TSIM: print =============== step8 + tdLog.info('=============== step8') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt group by tbcol + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s by tbcol" % mt) + tdSql.query("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tbcol" % mt) + #TSIM: + #TSIM: print =============== step9 + tdLog.info('=============== step9') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt group by noexist -x step9 + tdLog.info("select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by noexist -x step9" % mt) + tdSql.error('select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by noexist ' % mt) + #TSIM: return -1 + #TSIM: step9: + #TSIM: + #TSIM: print =============== step10 + tdLog.info('=============== step10') + #TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info('select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % mt) + tdSql.query('select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % mt) + #TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + #TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: print =============== step11 + tdLog.info('=============== step11') + #TSIM: sql select count(tbcol) as c from $mt group by tbcol + tdLog.info('select count(tbcol) as c from %s group by tbcol' % mt) + tdSql.query('select count(tbcol) as c from %s group by tbcol' % mt) + #TSIM: + #TSIM: print =============== step12 + tdLog.info('=============== step12') + #TSIM: sql select count(tbcol) as c from $mt group by noexist -x step12 + tdLog.info('select count(tbcol) as c from %s group by noexist -x step12' % mt) + tdSql.error('select count(tbcol) as c from %s group by noexist2' % mt) + #TSIM: return -1 + #TSIM: step12: + #TSIM: + #TSIM: print =============== step13 + tdLog.info('=============== step13') + #TSIM: sql select count(tbcol) as c from $mt group by tgcol + tdLog.info('select count(tbcol) as c from %s group by tgcol' % mt) + tdSql.query('select count(tbcol) as c from %s group by tgcol' % mt) + #TSIM: print $data00 + tdLog.info('$data00') + #TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: print =============== step14 + tdLog.info('=============== step14') + #TSIM: sql select count(tbcol) as c from $mt where ts > 1000 group by tgcol + tdLog.info('select count(tbcol) as c from %s where ts > 1000 group by tgcol' % mt) + tdSql.query('select count(tbcol) as c from %s where ts > 1000 group by tgcol' % mt) + #TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 +# tdLog.info("%s %s %s %s %s %s %s" % (tdSql.getData(0, 0), tdSql.getData(0, 1), tdSql.getData(0, 2), tdSql.getData(0, 3), tdSql.getData(0, 4), tdSql.getData(0, 5), tdSql.getData(0, 6))) + #TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + #TSIM: print expect 100, actual $data00 + tdLog.info('expect 100, actual $data00') + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: print =============== step15 + tdLog.info('=============== step15') + #TSIM: sql select count(tbcol) as c from $mt where noexist < 1 group by tgcol -x step15 + tdLog.info('select count(tbcol) as c from %s where noexist < 1 group by tgcol -x step15' % mt) + tdSql.error('select count(tbcol) as c from %s where noexist < 1 group by tgcol5' % mt) + #TSIM: return -1 + #TSIM: step15: + #TSIM: + #TSIM: print =============== step16 + tdLog.info('=============== step16') + #TSIM: sql select count(tbcol) as c from $mt where tgcol = '1' group by tgcol + tdLog.info("select count(tbcol) as c from %s where tgcol = '1' group by tgcol" % mt) + tdSql.query("select count(tbcol) as c from %s where tgcol = '1' group by tgcol" % mt) + #TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 +# tdLog.info("%s %s %s %s %s %s %s" % (tdSql.getData(0, 0), tdSql.getData(0, 1), tdSql.getData(0, 2), tdSql.getData(0, 3), tdSql.getData(0, 4), tdSql.getData(0, 5), tdSql.getData(0, 6))) + #TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: print =============== clear + tdLog.info('=============== clear') + #TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + #TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + #TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + #TSIM: return -1 + #TSIM: endi + #TSIM: + #TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 75f3bd9044..1cc0eddbfc 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -49,10 +49,10 @@ class TDSql: callerModule = inspect.getmodule(frame[0]) callerFilename = callerModule.__file__ tdLog.exit( - "%s failed: sql:%.40s, expect error not occured" % + "%s failed: sql:%s, expect error not occured" % (callerFilename, sql)) else: - tdLog.info("sql:%.40s, expect error occured" % (sql)) + tdLog.info("sql:%s, expect error occured" % (sql)) def query(self, sql): self.sql = sql @@ -72,9 +72,9 @@ class TDSql: callerModule = inspect.getmodule(frame[0]) callerFilename = callerModule.__file__ tdLog.exit( - "%s failed: sql:%.40s, queryRows:%d != expect:%d" % + "%s failed: sql:%s, queryRows:%d != expect:%d" % (callerFilename, self.sql, self.queryRows, expectRows)) - tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) def checkData(self, row, col, data): @@ -84,35 +84,35 @@ class TDSql: if row < 0: tdLog.exit( - "%s failed: sql:%.40s, row:%d is smaller than zero" % + "%s failed: sql:%s, row:%d is smaller than zero" % (callerFilename, self.sql, row)) if col < 0: tdLog.exit( - "%s failed: sql:%.40s, col:%d is smaller than zero" % + "%s failed: sql:%s, col:%d is smaller than zero" % (callerFilename, self.sql, col)) - if row >= self.queryRows: + if row > self.queryRows: tdLog.exit( - "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" % + "%s failed: sql:%s, row:%d is larger than queryRows:%d" % (callerFilename, self.sql, row, self.queryRows)) - if col >= self.queryCols: + if col > self.queryCols: tdLog.exit( - "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" % + "%s failed: sql:%s, col:%d is larger than queryCols:%d" % (callerFilename, self.sql, col, self.queryCols)) if self.queryResult[row][col] != data: - tdLog.exit("%s failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % ( + tdLog.exit("%s failed: sql:%s row:%d col:%d data:%s != expect:%s" % ( callerFilename, self.sql, row, col, self.queryResult[row][col], data)) if data is None: - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) elif isinstance(data, str): - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) elif isinstance(data, datetime.date): - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) else: - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % (self.sql, row, col, self.queryResult[row][col], data)) def getData(self, row, col): @@ -122,19 +122,19 @@ class TDSql: if row < 0: tdLog.exit( - "%s failed: sql:%.40s, row:%d is smaller than zero" % + "%s failed: sql:%s, row:%d is smaller than zero" % (callerFilename, self.sql, row)) if col < 0: tdLog.exit( - "%s failed: sql:%.40s, col:%d is smaller than zero" % + "%s failed: sql:%s, col:%d is smaller than zero" % (callerFilename, self.sql, col)) - if row >= self.queryRows: + if row > self.queryRows: tdLog.exit( - "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" % + "%s failed: sql:%s, row:%d is larger than queryRows:%d" % (callerFilename, self.sql, row, self.queryRows)) - if col >= self.queryCols: + if col > self.queryCols: tdLog.exit( - "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" % + "%s failed: sql:%s, col:%d is larger than queryCols:%d" % (callerFilename, self.sql, col, self.queryCols)) return self.queryResult[row][col] @@ -157,9 +157,9 @@ class TDSql: callerModule = inspect.getmodule(frame[0]) callerFilename = callerModule.__file__ - tdLog.exit("%s failed: sql:%.40s, affectedRows:%d != expect:%d" % ( + tdLog.exit("%s failed: sql:%s, affectedRows:%d != expect:%d" % ( callerFilename, self.sql, self.affectedRows, expectAffectedRows)) - tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" % + tdLog.info("sql:%s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) diff --git a/tests/pytest/valgrind-test.sh b/tests/pytest/valgrind-test.sh index 0b5dfc0fa3..55bb03c966 100755 --- a/tests/pytest/valgrind-test.sh +++ b/tests/pytest/valgrind-test.sh @@ -1,4 +1,5 @@ #!/bin/bash +# insert python3 ./test.py -g -f insert/basic.py python3 ./test.py -g -s && sleep 1 python3 ./test.py -g -f insert/int.py @@ -24,6 +25,7 @@ python3 ./test.py -g -s && sleep 1 python3 ./test.py -g -f insert/multi.py python3 ./test.py -g -s && sleep 1 +# table python3 ./test.py -g -f table/column_name.py python3 ./test.py -g -s && sleep 1 python3 ./test.py -g -f table/column_num.py @@ -31,5 +33,10 @@ python3 ./test.py -g -s && sleep 1 python3 ./test.py -g -f table/db_table.py python3 ./test.py -g -s && sleep 1 +# import python3 ./test.py -g -f import_merge/importDataLastSub.py python3 ./test.py -g -s && sleep 1 + +#tag +python3 ./test.py $1 -f tag_lite/filter.py +python3 ./test.py $1 -s && sleep 1 From 499170d3cf48661ea60ab478326e6264d55e9a81 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 May 2020 19:23:42 +0800 Subject: [PATCH 4/8] [TD-361] fix bug in show vgroups --- src/mnode/inc/mgmtDef.h | 1 - src/mnode/inc/mgmtVgroup.h | 5 ----- src/mnode/src/mgmtVgroup.c | 15 ++------------ tests/script/unique/big/maxvnodes.sim | 28 ++++++++++----------------- 4 files changed, 12 insertions(+), 37 deletions(-) diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 7c1fd6c5ab..f0d694db2a 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -123,7 +123,6 @@ typedef struct SVgObj { int32_t numOfVnodes; int32_t lbDnodeId; int32_t lbTime; - int8_t status; int8_t inUse; int8_t reserved[13]; int8_t updateEnd[1]; diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 7acf7112a4..ce4cfed532 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -22,11 +22,6 @@ extern "C" { #include "mgmtDef.h" -enum _TSDB_VG_STATUS { - TSDB_VG_STATUS_READY, - TSDB_VG_STATUS_UPDATE -}; - int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 1416ed5f58..ef5582f395 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -371,12 +371,6 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup_status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - int32_t maxReplica = 0; SVgObj *pVgroup = NULL; STableObj *pTable = NULL; @@ -471,11 +465,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo *(int32_t *) pWrite = pVgroup->numOfTables; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char* status = pVgroup->status? "updating" : "ready"; - STR_TO_VARSTR(pWrite, status); - cols++; - for (int32_t i = 0; i < maxReplica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; @@ -489,8 +478,8 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - status = mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role); - STR_TO_VARSTR(pWrite, status); + char *role = mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role); + STR_TO_VARSTR(pWrite, role); cols++; } else { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/tests/script/unique/big/maxvnodes.sim b/tests/script/unique/big/maxvnodes.sim index 3015d07b6d..2b56f51e59 100644 --- a/tests/script/unique/big/maxvnodes.sim +++ b/tests/script/unique/big/maxvnodes.sim @@ -1,19 +1,18 @@ system sh/stop_dnodes.sh $totalVnodes = 100 -$minVnodes = 48 -$maxVnodes = 52 +$minVnodes = 50 +$maxVnodes = 50 $maxTables = 4 $totalRows = $totalVnodes * $maxTables system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $maxTables +system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v $totalVnodes -system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000 -system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000 -system sh/cfg.sh -n dnode1 -c maxShellConns -v 100000 -system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 100000 +system sh/deploy.sh -n dnode2 -i 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v $totalVnodes + print ========== prepare data system sh/exec_up.sh -n dnode1 -s start @@ -44,16 +43,7 @@ if $data00 != $totalRows then return -1 endi -system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 100 -system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256 -system sh/cfg.sh -n dnode2 -c maxVnodeConnections -v 100000 -system sh/cfg.sh -n dnode2 -c maxMeterConnections -v 100000 -system sh/cfg.sh -n dnode2 -c maxShellConns -v 100000 -system sh/cfg.sh -n dnode2 -c maxMgmtConnections -v 100000 - -print ========== step2 +print ========== step3 sql create dnode $hostname2 system sh/exec_up.sh -n dnode2 -s start @@ -86,6 +76,8 @@ if $data00 != $totalRows then return -1 endi +return + system sh/exec_up.sh -n dnode1 -s stop -x SIGINT system sh/exec_up.sh -n dnode2 -s stop -x SIGINT system sh/exec_up.sh -n dnode3 -s stop -x SIGINT From 908aa37a416a2d5d87d843144b18dff7030e1a14 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 18 May 2020 14:15:21 +0000 Subject: [PATCH 5/8] add new macro TSDB_EP_LEN --- src/client/src/tscUtil.c | 4 +- src/common/src/tglobal.c | 14 +- src/dnode/src/dnodeMgmt.c | 2 +- src/inc/taosdef.h | 3 +- src/inc/taosmsg.h | 10 +- src/mnode/inc/mgmtDef.h | 2 +- src/plugins/monitor/src/monitorMain.c | 2 +- src/rpc/inc/rpcHaship.h | 33 ---- src/rpc/src/rpcHaship.c | 167 ------------------ src/rpc/src/rpcUdp.c | 234 +++----------------------- 10 files changed, 43 insertions(+), 428 deletions(-) delete mode 100644 src/rpc/inc/rpcHaship.h delete mode 100644 src/rpc/src/rpcHaship.c diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4667606aa8..8570c2b304 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2169,7 +2169,7 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { tscMgmtIpSet.inUse = 0; if (first && first[0] != 0) { - if (strlen(first) >= TSDB_FQDN_LEN) { + if (strlen(first) >= TSDB_EP_LEN) { terrno = TSDB_CODE_INVALID_FQDN; return -1; } @@ -2178,7 +2178,7 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { } if (second && second[0] != 0) { - if (strlen(second) >= TSDB_FQDN_LEN) { + if (strlen(second) >= TSDB_EP_LEN) { terrno = TSDB_CODE_INVALID_FQDN; return -1; } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 18d8c9ebe2..324edb422b 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -61,10 +61,10 @@ int32_t tscEmbedded = 0; */ int64_t tsMsPerDay[] = {86400000L, 86400000000L}; -char tsFirst[TSDB_FQDN_LEN] = {0}; -char tsSecond[TSDB_FQDN_LEN] = {0}; -char tsArbitrator[TSDB_FQDN_LEN] = {0}; -char tsLocalEp[TSDB_FQDN_LEN] = {0}; // Local End Point, hostname:port +char tsFirst[TSDB_EP_LEN] = {0}; +char tsSecond[TSDB_EP_LEN] = {0}; +char tsArbitrator[TSDB_EP_LEN] = {0}; +char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port uint16_t tsServerPort = 6030; uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035] uint16_t tsDnodeDnodePort = 6035; // udp/tcp @@ -284,7 +284,7 @@ static void doInitGlobalConfig() { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; + cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -294,7 +294,7 @@ static void doInitGlobalConfig() { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; + cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -356,7 +356,7 @@ static void doInitGlobalConfig() { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; + cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 81d426b496..4b28992aa4 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -411,7 +411,7 @@ static bool dnodeReadMnodeInfos() { dError("failed to read mnode mgmtIpList.json, nodeName not found"); goto PARSE_OVER; } - strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN); + strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); } ret = true; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ba015d7bbf..2393654f79 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -218,7 +218,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 64 -#define TSDB_FQDN_LEN 256 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN+6) #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 #define TSDB_METER_VNODE_BITS 20 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 27862ef489..3aa75523ea 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -530,7 +530,7 @@ typedef struct { typedef struct { int32_t nodeId; - char nodeEp[TSDB_FQDN_LEN]; + char nodeEp[TSDB_EP_LEN]; } SDMMnodeInfo; typedef struct { @@ -542,7 +542,7 @@ typedef struct { typedef struct { uint32_t version; int32_t dnodeId; - char dnodeEp[TSDB_FQDN_LEN]; + char dnodeEp[TSDB_EP_LEN]; uint32_t moduleStatus; uint32_t lastReboot; // time stamp for last reboot uint16_t numOfTotalVnodes; // from config file @@ -584,7 +584,7 @@ typedef struct { typedef struct { int32_t nodeId; - char nodeEp[TSDB_FQDN_LEN]; + char nodeEp[TSDB_EP_LEN]; } SMDVnodeDesc; typedef struct { @@ -669,7 +669,7 @@ typedef struct SCMShowRsp { } SCMShowRsp; typedef struct { - char ep[TSDB_FQDN_LEN]; // end point, hostname:port + char ep[TSDB_EP_LEN]; // end point, hostname:port } SCMCreateDnodeMsg, SCMDropDnodeMsg; typedef struct { @@ -684,7 +684,7 @@ typedef struct { } SDMConfigVnodeMsg; typedef struct { - char ep[TSDB_FQDN_LEN]; // end point, hostname:port + char ep[TSDB_EP_LEN]; // end point, hostname:port char config[64]; } SMDCfgDnodeMsg, SCMCfgDnodeMsg; diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index f0d694db2a..fac342901a 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -33,7 +33,7 @@ typedef struct SDnodeObj { int32_t dnodeId; uint16_t dnodePort; char dnodeFqdn[TSDB_FQDN_LEN + 1]; - char dnodeEp[TSDB_FQDN_LEN + 1]; + char dnodeEp[TSDB_EP_LEN + 1]; int64_t createdTime; uint32_t lastAccess; int32_t openVnodes; diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index ae8038e444..72efd5b552 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -68,7 +68,7 @@ typedef enum { typedef struct { void * conn; void * timer; - char ep[TSDB_FQDN_LEN]; + char ep[TSDB_EP_LEN]; int8_t cmdIndex; int8_t state; char sql[SQL_LENGTH]; diff --git a/src/rpc/inc/rpcHaship.h b/src/rpc/inc/rpcHaship.h deleted file mode 100644 index d3ed48997a..0000000000 --- a/src/rpc/inc/rpcHaship.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _rpc_hash_ip_header_ -#define _rpc_hash_ip_header_ - -#ifdef __cplusplus -extern "C" { -#endif - -void *rpcOpenIpHash(int maxSessions); -void rpcCloseIpHash(void *handle); -void *rpcAddIpHash(void *handle, void *pData, uint32_t ip, uint16_t port); -void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port); -void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/rpc/src/rpcHaship.c b/src/rpc/src/rpcHaship.c deleted file mode 100644 index 0183c87f70..0000000000 --- a/src/rpc/src/rpcHaship.c +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmempool.h" -#include "rpcLog.h" - -typedef struct SIpHash { - uint32_t ip; - uint16_t port; - int hash; - struct SIpHash *prev; - struct SIpHash *next; - void *data; -} SIpHash; - -typedef struct { - SIpHash **ipHashList; - mpool_h ipHashMemPool; - int maxSessions; -} SHashObj; - -int rpcHashIp(void *handle, uint32_t ip, uint16_t port) { - SHashObj *pObj = (SHashObj *)handle; - int hash = 0; - - hash = (int)(ip >> 16); - hash += (unsigned short)(ip & 0xFFFF); - hash += port; - - hash = hash % pObj->maxSessions; - - return hash; -} - -void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { - int hash; - SIpHash *pNode; - SHashObj *pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = rpcHashIp(pObj, ip, port); - pNode = (SIpHash *)taosMemPoolMalloc(pObj->ipHashMemPool); - pNode->ip = ip; - pNode->port = port; - pNode->data = data; - pNode->prev = 0; - pNode->next = pObj->ipHashList[hash]; - pNode->hash = hash; - - if (pObj->ipHashList[hash] != 0) (pObj->ipHashList[hash])->prev = pNode; - pObj->ipHashList[hash] = pNode; - - return pObj; -} - -void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { - int hash; - SIpHash *pNode; - SHashObj *pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - hash = rpcHashIp(pObj, ip, port); - - pNode = pObj->ipHashList[hash]; - while (pNode) { - if (pNode->ip == ip && pNode->port == port) break; - - pNode = pNode->next; - } - - if (pNode) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->ipHashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - taosMemPoolFree(pObj->ipHashMemPool, (char *)pNode); - } -} - -void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) { - int hash; - SIpHash *pNode; - SHashObj *pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = rpcHashIp(pObj, ip, port); - pNode = pObj->ipHashList[hash]; - - while (pNode) { - if (pNode->ip == ip && pNode->port == port) { - break; - } - pNode = pNode->next; - } - - if (pNode) { - return pNode->data; - } - return NULL; -} - -void *rpcOpenIpHash(int maxSessions) { - SIpHash **ipHashList; - mpool_h ipHashMemPool; - SHashObj *pObj; - - ipHashMemPool = taosMemPoolInit(maxSessions, sizeof(SIpHash)); - if (ipHashMemPool == 0) return NULL; - - ipHashList = calloc(sizeof(SIpHash *), (size_t)maxSessions); - if (ipHashList == 0) { - taosMemPoolCleanUp(ipHashMemPool); - return NULL; - } - - pObj = malloc(sizeof(SHashObj)); - if (pObj == NULL) { - taosMemPoolCleanUp(ipHashMemPool); - free(ipHashList); - return NULL; - } - - pObj->maxSessions = maxSessions; - pObj->ipHashMemPool = ipHashMemPool; - pObj->ipHashList = ipHashList; - - return pObj; -} - -void rpcCloseIpHash(void *handle) { - SHashObj *pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - if (pObj->ipHashMemPool) taosMemPoolCleanUp(pObj->ipHashMemPool); - - if (pObj->ipHashList) free(pObj->ipHashList); - - memset(pObj, 0, sizeof(SHashObj)); - free(pObj); -} diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 677187e3b9..3a40f27e26 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -19,7 +19,6 @@ #include "ttimer.h" #include "tutil.h" #include "rpcLog.h" -#include "rpcHaship.h" #include "rpcUdp.h" #include "rpcHead.h" @@ -28,8 +27,6 @@ #define RPC_UDP_BUF_TIME 5 // mseconds #define RPC_MAX_UDP_SIZE 65480 -int tsUdpDelay = 0; - typedef struct { void *signature; int index; @@ -38,8 +35,6 @@ typedef struct { uint16_t localPort; // local port char label[12]; // copy from udpConnSet; pthread_t thread; - pthread_mutex_t mutex; - void *tmrCtrl; // copy from UdpConnSet; void *hash; void *shandle; // handle passed by upper layer during server initialization void *pSet; @@ -55,26 +50,11 @@ typedef struct { void *shandle; // handle passed by upper layer during server initialization int threads; char label[12]; - void *tmrCtrl; void *(*fp)(SRecvInfo *pPacket); SUdpConn udpConn[]; } SUdpConnSet; -typedef struct { - void *signature; - uint32_t ip; // dest IP - uint16_t port; // dest Port - SUdpConn *pConn; - struct sockaddr_in destAdd; - void *msgHdr; - int totalLen; - void *timer; - int emptyNum; -} SUdpBuf; - static void *taosRecvUdpData(void *param); -static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); -static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { SUdpConn *pConn; @@ -94,16 +74,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pSet->fp = fp; strcpy(pSet->label, label); - if ( tsUdpDelay ) { - char udplabel[12]; - sprintf(udplabel, "%s.b", label); - pSet->tmrCtrl = taosTmrInit(RPC_MAX_UDP_CONNS * threads, 5, 5000, udplabel); - if (pSet->tmrCtrl == NULL) { - tError("%s failed to initialize tmrCtrl") taosCleanUpUdpConnection(pSet); - return NULL; - } - } - uint16_t ownPort; for (int i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -135,11 +105,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pConn->index = i; pConn->pSet = pSet; pConn->signature = pConn; - if (tsUdpDelay) { - pConn->hash = rpcOpenIpHash(RPC_MAX_UDP_CONNS); - pthread_mutex_init(&pConn->mutex, NULL); - pConn->tmrCtrl = pSet->tmrCtrl; - } pthread_attr_t thAttr; pthread_attr_init(&thAttr); @@ -173,10 +138,6 @@ void taosCleanUpUdpConnection(void *handle) { free(pConn->buffer); pthread_cancel(pConn->thread); taosCloseSocket(pConn->fd); - if (pConn->hash) { - rpcCloseIpHash(pConn->hash); - pthread_mutex_destroy(&pConn->mutex); - } } for (int i = 0; i < pSet->threads; ++i) { @@ -185,7 +146,6 @@ void taosCleanUpUdpConnection(void *handle) { tTrace("chandle:%p is closed", pConn); } - taosTmrCleanUp(pSet->tmrCtrl); tfree(pSet); } @@ -205,64 +165,42 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t static void *taosRecvUdpData(void *param) { SUdpConn *pConn = param; struct sockaddr_in sourceAdd; - int dataLen; + ssize_t dataLen; unsigned int addLen; uint16_t port; - int minSize = sizeof(SRpcHead); SRecvInfo recvInfo; memset(&sourceAdd, 0, sizeof(sourceAdd)); addLen = sizeof(sourceAdd); tTrace("%s UDP thread is created, index:%d", pConn->label, pConn->index); + char *msg = pConn->buffer; while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); port = ntohs(sourceAdd.sin_port); - //tTrace("%s msg is recv from 0x%x:%hu len:%d", pConn->label, sourceAdd.sin_addr.s_addr, port, dataLen); if (dataLen < sizeof(SRpcHead)) { tError("%s recvfrom failed, reason:%s\n", pConn->label, strerror(errno)); continue; } - int processedLen = 0, leftLen = 0; - int msgLen = 0; - int count = 0; - char *msg = pConn->buffer; - while (processedLen < dataLen) { - leftLen = dataLen - processedLen; - SRpcHead *pHead = (SRpcHead *)msg; - msgLen = htonl((uint32_t)pHead->msgLen); - if (leftLen < minSize || msgLen > leftLen || msgLen < minSize) { - tError("%s msg is messed up, dataLen:%d processedLen:%d count:%d msgLen:%d", pConn->label, dataLen, - processedLen, count, msgLen); - break; - } - - char *tmsg = malloc((size_t)msgLen + tsRpcOverhead); - if (NULL == tmsg) { - tError("%s failed to allocate memory, size:%d", pConn->label, msgLen); - break; - } - - tmsg += tsRpcOverhead; // overhead for SRpcReqContext - memcpy(tmsg, msg, (size_t)msgLen); - recvInfo.msg = tmsg; - recvInfo.msgLen = msgLen; - recvInfo.ip = sourceAdd.sin_addr.s_addr; - recvInfo.port = port; - recvInfo.shandle = pConn->shandle; - recvInfo.thandle = NULL; - recvInfo.chandle = pConn; - recvInfo.connType = 0; - (*(pConn->processData))(&recvInfo); - - processedLen += msgLen; - msg += msgLen; - count++; + char *tmsg = malloc(dataLen + tsRpcOverhead); + if (NULL == tmsg) { + tError("%s failed to allocate memory, size:%d", pConn->label, dataLen); + continue; } - // tTrace("%s %d UDP packets are received together", pConn->label, count); + tmsg += tsRpcOverhead; // overhead for SRpcReqContext + memcpy(tmsg, msg, dataLen); + recvInfo.msg = tmsg; + recvInfo.msgLen = dataLen; + recvInfo.ip = sourceAdd.sin_addr.s_addr; + recvInfo.port = port; + recvInfo.shandle = pConn->shandle; + recvInfo.thandle = NULL; + recvInfo.chandle = pConn; + recvInfo.connType = 0; + (*(pConn->processData))(&recvInfo); } return NULL; @@ -270,141 +208,17 @@ static void *taosRecvUdpData(void *param) { int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) { SUdpConn *pConn = (SUdpConn *)chandle; - SUdpBuf *pBuf; if (pConn == NULL || pConn->signature != pConn) return -1; - if (pConn->hash == NULL) { - struct sockaddr_in destAdd; - memset(&destAdd, 0, sizeof(destAdd)); - destAdd.sin_family = AF_INET; - destAdd.sin_addr.s_addr = ip; - destAdd.sin_port = htons(port); + struct sockaddr_in destAdd; + memset(&destAdd, 0, sizeof(destAdd)); + destAdd.sin_family = AF_INET; + destAdd.sin_addr.s_addr = ip; + destAdd.sin_port = htons(port); - //tTrace("%s msg is sent to 0x%x:%hu len:%d ret:%d localPort:%hu chandle:0x%x", pConn->label, destAdd.sin_addr.s_addr, - // port, dataLen, ret, pConn->localPort, chandle); - int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); + int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); - return ret; - } - - pthread_mutex_lock(&pConn->mutex); - - pBuf = (SUdpBuf *)rpcGetIpHash(pConn->hash, ip, port); - if (pBuf == NULL) { - pBuf = taosCreateUdpBuf(pConn, ip, port); - rpcAddIpHash(pConn->hash, pBuf, ip, port); - } - - if ((pBuf->totalLen + dataLen > RPC_MAX_UDP_SIZE) || (taosMsgHdrSize(pBuf->msgHdr) >= RPC_MAX_UDP_PKTS)) { - taosTmrReset(taosProcessUdpBufTimer, RPC_UDP_BUF_TIME, pBuf, pConn->tmrCtrl, &pBuf->timer); - - taosSendMsgHdr(pBuf->msgHdr, pConn->fd); - pBuf->totalLen = 0; - } - - taosSetMsgHdrData(pBuf->msgHdr, data, dataLen); - - pBuf->totalLen += dataLen; - - pthread_mutex_unlock(&pConn->mutex); - - return dataLen; + return ret; } -void taosFreeMsgHdr(void *hdr) { - struct msghdr *msgHdr = (struct msghdr *)hdr; - free(msgHdr->msg_iov); -} - -int taosMsgHdrSize(void *hdr) { - struct msghdr *msgHdr = (struct msghdr *)hdr; - return (int)msgHdr->msg_iovlen; -} - -void taosSendMsgHdr(void *hdr, int fd) { - struct msghdr *msgHdr = (struct msghdr *)hdr; - sendmsg(fd, msgHdr, 0); - msgHdr->msg_iovlen = 0; -} - -void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) { - struct msghdr *msgHdr = (struct msghdr *)malloc(sizeof(struct msghdr)); - memset(msgHdr, 0, sizeof(struct msghdr)); - *hdr = msgHdr; - struct sockaddr_in *destAdd = (struct sockaddr_in *)dest; - - msgHdr->msg_name = destAdd; - msgHdr->msg_namelen = sizeof(struct sockaddr_in); - int size = (int)sizeof(struct iovec) * maxPkts; - msgHdr->msg_iov = (struct iovec *)malloc((size_t)size); - memset(msgHdr->msg_iov, 0, (size_t)size); -} - -void taosSetMsgHdrData(void *hdr, char *data, int dataLen) { - struct msghdr *msgHdr = (struct msghdr *)hdr; - msgHdr->msg_iov[msgHdr->msg_iovlen].iov_base = data; - msgHdr->msg_iov[msgHdr->msg_iovlen].iov_len = (size_t)dataLen; - msgHdr->msg_iovlen++; -} - -void taosRemoveUdpBuf(SUdpBuf *pBuf) { - taosTmrStopA(&pBuf->timer); - rpcDeleteIpHash(pBuf->pConn->hash, pBuf->ip, pBuf->port); - - // tTrace("%s UDP buffer to:0x%lld:%d is removed", pBuf->pConn->label, - // pBuf->ip, pBuf->port); - - pBuf->signature = NULL; - taosFreeMsgHdr(pBuf->msgHdr); - free(pBuf); -} - -void taosProcessUdpBufTimer(void *param, void *tmrId) { - SUdpBuf *pBuf = (SUdpBuf *)param; - if (pBuf->signature != param) return; - if (pBuf->timer != tmrId) return; - - SUdpConn *pConn = pBuf->pConn; - - pthread_mutex_lock(&pConn->mutex); - - if (taosMsgHdrSize(pBuf->msgHdr) > 0) { - taosSendMsgHdr(pBuf->msgHdr, pConn->fd); - pBuf->totalLen = 0; - pBuf->emptyNum = 0; - } else { - pBuf->emptyNum++; - if (pBuf->emptyNum > 200) { - taosRemoveUdpBuf(pBuf); - pBuf = NULL; - } - } - - pthread_mutex_unlock(&pConn->mutex); - - if (pBuf) taosTmrReset(taosProcessUdpBufTimer, RPC_UDP_BUF_TIME, pBuf, pConn->tmrCtrl, &pBuf->timer); -} - -static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port) { - SUdpBuf *pBuf = (SUdpBuf *)malloc(sizeof(SUdpBuf)); - memset(pBuf, 0, sizeof(SUdpBuf)); - - pBuf->ip = ip; - pBuf->port = port; - pBuf->pConn = pConn; - - pBuf->destAdd.sin_family = AF_INET; - pBuf->destAdd.sin_addr.s_addr = ip; - pBuf->destAdd.sin_port = (uint16_t)htons(port); - taosInitMsgHdr(&(pBuf->msgHdr), &(pBuf->destAdd), RPC_MAX_UDP_PKTS); - pBuf->signature = pBuf; - taosTmrReset(taosProcessUdpBufTimer, RPC_UDP_BUF_TIME, pBuf, pConn->tmrCtrl, &pBuf->timer); - - // tTrace("%s UDP buffer to:0x%lld:%d is created", pBuf->pConn->label, - // pBuf->ip, pBuf->port); - - return pBuf; -} - - From 6b2702d2e71dcf7eb832be1eac389a1bdfd3ffb1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 May 2020 22:23:33 +0800 Subject: [PATCH 6/8] scrpits --- tests/script/general/db/basic4.sim | 6 ------ tests/script/sh/deploy.sh | 8 ++++---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/script/general/db/basic4.sim b/tests/script/general/db/basic4.sim index 8494b0358a..a0a9aaa627 100644 --- a/tests/script/general/db/basic4.sim +++ b/tests/script/general/db/basic4.sim @@ -43,9 +43,6 @@ endi if $data01 != 4 then return -1 endi -if $data02 != ready then - return -1 -endi print =============== drop table sql drop table d1.t1 @@ -82,9 +79,6 @@ endi if $data01 != 3 then return -1 endi -if $data02 != ready then - return -1 -endi print =============== drop all table sql drop table d1.t2 diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index bc18479896..bd85ce9704 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -96,10 +96,10 @@ echo "second ${HOSTNAME}:7200" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "dDebugFlag 199" >> $TAOS_CFG -echo "mDebugFlag 199" >> $TAOS_CFG -echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 151" >> $TAOS_CFG +echo "dDebugFlag 135" >> $TAOS_CFG +echo "mDebugFlag 135" >> $TAOS_CFG +echo "sdbDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 135" >> $TAOS_CFG From 6a3043753d040c5221e7981842074eb28518e9b0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 May 2020 15:34:19 +0000 Subject: [PATCH 7/8] [TD-362] fix crash while auto create table --- src/mnode/src/mgmtTable.c | 7 ++++++- tests/script/general/db/basic5.sim | 3 --- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index de8f2f0ab0..3043306978 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -1415,7 +1415,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj char *pTagData = (char *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); if (pSuperTable == NULL) { - mError("table:%s, corresponding super table does not exist", pCreate->tableId); + mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData); free(pTable); terrno = TSDB_CODE_INVALID_TABLE; return NULL; @@ -1505,6 +1505,11 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { } pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); + if (pMsg->pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, terrno); + return; + } + mgmtIncTableRef(pMsg->pTable); } } else { diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index 3c59144387..82b9bf9bf4 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -43,9 +43,6 @@ endi if $data01 != 4 then return -1 endi -if $data02 != ready then - return -1 -endi print =============== drop database sql drop database d1 From 34b252501a7dc191f71b7af595fa2e5c84db51ee Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 May 2020 15:42:36 +0000 Subject: [PATCH 8/8] scripts --- tests/script/general/table/column_value.sim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/script/general/table/column_value.sim b/tests/script/general/table/column_value.sim index bd98d3b290..72f5faee63 100644 --- a/tests/script/general/table/column_value.sim +++ b/tests/script/general/table/column_value.sim @@ -34,13 +34,12 @@ if $rows != 0 then endi print =============== step2 -sql create table $tb (ts timestamp, speed bigint, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) -x step2 - return -1 -step2: +sql create table $tb (ts timestamp, speed bigint, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) sql show tables -if $rows != 0 then +if $rows != 1 then return -1 endi +sql drop table $tb print =============== step3 sql create table $tb (ts timestamp, speed float, v1 binary(100), v2 binary(100), v3 binary(100), v4 binary(100), v5 binary(100))