From 84cc10bf28b413420712394c09acaa1aa7e3b474 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 May 2020 18:33:06 +0800 Subject: [PATCH 1/8] add max column number check case. --- tests/pytest/table/column_num.py | 109 ++++++++----------------------- 1 file changed, 27 insertions(+), 82 deletions(-) diff --git a/tests/pytest/table/column_num.py b/tests/pytest/table/column_num.py index 87299827cc..6e9b6ca353 100644 --- a/tests/pytest/table/column_num.py +++ b/tests/pytest/table/column_num.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import sys +import subprocess from util.log import * from util.cases import * from util.sql import * @@ -14,127 +15,71 @@ class TDTestCase: def run(self): tdSql.prepare() - # TSIM: system sh/stop_dnodes.sh - # TSIM: - # TSIM: system sh/ip.sh -i 1 -s up - # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 - # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 - # TSIM: system sh/exec.sh -n dnode1 -s start - # TSIM: - # TSIM: sleep 3000 - # TSIM: sql connect - # TSIM: - # TSIM: $i = 0 - # TSIM: $dbPrefix = lm_cn_db - # TSIM: $tbPrefix = lm_cn_tb - # TSIM: $db = $dbPrefix . $i - # TSIM: $tb = $tbPrefix . $i - # TSIM: - # TSIM: print =============== step1 tdLog.info('=============== step1') - # TSIM: sql create database $db - # TSIM: sql use $db - # TSIM: - # TSIM: sql create table $tb() -x step1 tdLog.info('create table tb() -x step1') tdSql.error('create table tb()') - # TSIM: return -1 - # TSIM: step1: - # TSIM: - # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') # TSIM: if $rows != 0 then tdLog.info('tdSql.checkRow(0)') tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step2 tdLog.info('=============== step2') # TSIM: sql create table $tb (ts timestamp) -x step2 tdLog.info('create table tb (ts timestamp) -x step2') tdSql.error('create table tb (ts timestamp) ') - # TSIM: return -1 - # TSIM: step2: - # TSIM: - # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') # TSIM: if $rows != 0 then tdLog.info('tdSql.checkRow(0)') tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step3 tdLog.info('=============== step3') - # TSIM: sql create table $tb (ts int) -x step3 tdLog.info('create table tb (ts int) -x step3') tdSql.error('create table tb (ts int) ') - # TSIM: return -1 - # TSIM: step3: - # TSIM: - # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') - # TSIM: if $rows != 0 then tdLog.info('tdSql.checkRow(0)') tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step4 tdLog.info('=============== step4') - # TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4 - # int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 - # int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, - # a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 - # int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, - # b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 - # int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, - # b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 - # int, b28 int,b29 int,b30 int,b31 int,b32 int) tdLog.info('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)') tdSql.execute('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)') # TSIM: # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') - # TSIM: if $rows != 1 then tdLog.info('tdSql.checkRow(1)') tdSql.checkRows(1) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step5 + tdLog.info('=============== step5') - # TSIM: $i = 1 - # TSIM: $tb = $tbPrefix . $i - # TSIM: - # TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4 - # int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 - # int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, - # a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 - # int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, - # b4 int, b5 int) - tdLog.info('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)') - tdSql.execute('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)') - # TSIM: - # TSIM: sql show tables + + getMaxColumnNum = "grep -w '#define TSDB_MAX_COLUMNS' ../../src/inc/taosdef.h|awk '{print $3}'" + boundary = int(subprocess.check_output(getMaxColumnNum, shell=True)) + tdLog.notice("get max column number is %d" % boundary) + + columnSeq = "ts timestamp" + for x in range(0, boundary): + columnSeq = columnSeq + ", col%d int" % x + + tdLog.info("create table tb1 (%s)" % columnSeq) + tdSql.error('create table tb1 (%s)' % columnSeq) + + columnSeq = "ts timestamp" + for x in range(0, boundary - 1): + columnSeq = columnSeq + ", col%d int" % x + + tdLog.info("create table tb1 (%s)" % columnSeq) + tdSql.execute('create table tb1 (%s)' % columnSeq) + tdLog.info('show tables') tdSql.query('show tables') # TSIM: if $rows != 2 then tdLog.info('tdSql.checkRow(2)') tdSql.checkRows(2) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: sql insert into $tb values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 - # , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 - # , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37) - tdLog.info("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)") - tdSql.execute("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)") + + data = "now" + for x in range(0, boundary-1): + data = data + ", %d" % x + tdLog.info("insert into tb1 values (%s)" % data) + tdSql.execute("insert into tb1 values (%s)" % data) # TSIM: sql select * from $tb tdLog.info('select * from tb1') tdSql.query('select * from tb1') From 6b8aae25a72a9f3d74dcb48b9085d66286debad5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 May 2020 18:40:12 +0800 Subject: [PATCH 2/8] change dist to bionicfor python 3.6+ --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 41937853d6..14d34ec970 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ # matrix: - os: linux + dist: bionic language: c git: From b4b092b48831df8d0492ae32ea9d769b165634fb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 May 2020 19:06:41 +0800 Subject: [PATCH 3/8] add random test. --- .travis.yml | 4 ++-- tests/pytest/dbmgmt/database-name-boundary.py | 2 +- tests/pytest/table/column_num.py | 2 +- tests/pytest/tag_lite/create-tags-boundary.py | 2 +- tests/pytest/util/dnodes.py | 10 ++-------- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b7ec6ffe7..da0c47ea56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,7 +62,7 @@ matrix: grep 'start to execute\|ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt - for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` + for memError in `grep 'ERROR SUMMARY' uniq-mem-error-out.txt | awk '{print $4}'` do if [ -n "$memError" ]; then if [ "$memError" -gt 12 ]; then @@ -74,7 +74,7 @@ matrix: done grep 'start to execute\|definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt - for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'` + for defiMemError in `grep 'definitely lost:' uniq-definitely-lost-out.txt | awk '{print $7}'` do if [ -n "$defiMemError" ]; then if [ "$defiMemError" -gt 13 ]; then diff --git a/tests/pytest/dbmgmt/database-name-boundary.py b/tests/pytest/dbmgmt/database-name-boundary.py index b57740ebaa..ff6dce22ae 100644 --- a/tests/pytest/dbmgmt/database-name-boundary.py +++ b/tests/pytest/dbmgmt/database-name-boundary.py @@ -33,7 +33,7 @@ class TDTestCase: getDbNameLen = "grep -w '#define TSDB_DB_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" dbNameMaxLen = int(subprocess.check_output(getDbNameLen, shell=True)) - tdLog.notice("DB name max length is %d" % dbNameMaxLen) + tdLog.info("DB name max length is %d" % dbNameMaxLen) tdLog.info("=============== step1") db_name = ''.join(random.choices(chars, k=(dbNameMaxLen + 1))) diff --git a/tests/pytest/table/column_num.py b/tests/pytest/table/column_num.py index 6e9b6ca353..486541bab7 100644 --- a/tests/pytest/table/column_num.py +++ b/tests/pytest/table/column_num.py @@ -53,7 +53,7 @@ class TDTestCase: getMaxColumnNum = "grep -w '#define TSDB_MAX_COLUMNS' ../../src/inc/taosdef.h|awk '{print $3}'" boundary = int(subprocess.check_output(getMaxColumnNum, shell=True)) - tdLog.notice("get max column number is %d" % boundary) + tdLog.info("get max column number is %d" % boundary) columnSeq = "ts timestamp" for x in range(0, boundary): diff --git a/tests/pytest/tag_lite/create-tags-boundary.py b/tests/pytest/tag_lite/create-tags-boundary.py index 12058b2b21..e80f458f0c 100644 --- a/tests/pytest/tag_lite/create-tags-boundary.py +++ b/tests/pytest/tag_lite/create-tags-boundary.py @@ -28,7 +28,7 @@ class TDTestCase: getMaxTagNum = "grep -w TSDB_MAX_TAGS ../../src/inc/taosdef.h|awk '{print $3}'" boundary = int(subprocess.check_output(getMaxTagNum, shell=True)) - tdLog.notice("get max tags number is %d" % boundary) + tdLog.info("get max tags number is %d" % boundary) for x in range(0, boundary): stb_name = "stb%d" % x diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0789c3bc3f..9f7466a7e7 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -244,11 +244,8 @@ class TDDnode: time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) + self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - tdLog.debug( - "wait 2 seconds for the dnode:%d to stop." % - (self.index)) - time.sleep(2) def forcestop(self): if self.valgrind == 0: @@ -267,11 +264,8 @@ class TDDnode: time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) + self.running = 0 tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - tdLog.debug( - "wait 2 seconds for the dnode:%d to stop." % - (self.index)) - time.sleep(2) def startIP(self): cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) From 53a0e401d48dadb9fea72416f6957b0fe6145e19 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 May 2020 09:52:05 +0800 Subject: [PATCH 4/8] gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d0af51d31b..83df761654 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,8 @@ Target/ *.failed *.sql sim/ +psim/ +pysim/ +*.out *DS_Store From 05a289d86631a0db53881bd0dc415f7ce5f1c251 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 16 May 2020 11:35:31 +0800 Subject: [PATCH 5/8] make failed msg friendly. --- tests/pytest/table/column_num.py | 2 +- tests/pytest/table/tablename-boundary.py | 2 +- tests/pytest/util/cases.py | 2 +- tests/pytest/util/sql.py | 68 +++++++++++++++--------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/tests/pytest/table/column_num.py b/tests/pytest/table/column_num.py index 486541bab7..877f0409dc 100644 --- a/tests/pytest/table/column_num.py +++ b/tests/pytest/table/column_num.py @@ -76,7 +76,7 @@ class TDTestCase: tdSql.checkRows(2) data = "now" - for x in range(0, boundary-1): + for x in range(0, boundary - 1): data = data + ", %d" % x tdLog.info("insert into tb1 values (%s)" % data) tdSql.execute("insert into tb1 values (%s)" % data) diff --git a/tests/pytest/table/tablename-boundary.py b/tests/pytest/table/tablename-boundary.py index fba1b2cf3c..335073065c 100644 --- a/tests/pytest/table/tablename-boundary.py +++ b/tests/pytest/table/tablename-boundary.py @@ -21,7 +21,7 @@ class TDTestCase: tableNameMaxLen = int( subprocess.check_output( getTableNameLen, shell=True)) - tdLog.notice("table name max length is %d" % tableNameMaxLen) + tdLog.info("table name max length is %d" % tableNameMaxLen) chars = string.ascii_uppercase + string.ascii_lowercase tb_name = ''.join(random.choices(chars, k=tableNameMaxLen)) tdLog.info('tb_name length %d' % len(tb_name)) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 9026cd0f2f..f65b0dfde3 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -71,7 +71,7 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) - tdLog.exit("%s failed: %s" % (__file__, fileName)) + tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 continue diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 0e7e186206..75f3bd9044 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +import inspect from util.log import * @@ -44,7 +45,12 @@ class TDSql: except BaseException: expectErrNotOccured = False if expectErrNotOccured: - tdLog.exit("failed: sql:%.40s, expect error not occured" % (sql)) + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + tdLog.exit( + "%s failed: sql:%.40s, expect error not occured" % + (callerFilename, sql)) else: tdLog.info("sql:%.40s, expect error occured" % (sql)) @@ -62,33 +68,39 @@ class TDSql: def checkRows(self, expectRows): if self.queryRows != expectRows: + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ tdLog.exit( - "failed: sql:%.40s, queryRows:%d != expect:%d" % - (self.sql, self.queryRows, expectRows)) + "%s failed: sql:%.40s, queryRows:%d != expect:%d" % + (callerFilename, self.sql, self.queryRows, expectRows)) tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) def checkData(self, row, col, data): + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + if row < 0: tdLog.exit( - "failed: sql:%.40s, row:%d is smaller than zero" % - (self.sql, row)) + "%s failed: sql:%.40s, row:%d is smaller than zero" % + (callerFilename, self.sql, row)) if col < 0: tdLog.exit( - "failed: sql:%.40s, col:%d is smaller than zero" % - (self.sql, col)) + "%s failed: sql:%.40s, col:%d is smaller than zero" % + (callerFilename, self.sql, col)) if row >= self.queryRows: tdLog.exit( - "failed: sql:%.40s, row:%d is larger than queryRows:%d" % - (self.sql, row, self.queryRows)) + "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" % + (callerFilename, self.sql, row, self.queryRows)) if col >= self.queryCols: tdLog.exit( - "failed: sql:%.40s, col:%d is larger than queryRows:%d" % - (self.sql, col, self.queryCols)) + "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" % + (callerFilename, self.sql, col, self.queryCols)) if self.queryResult[row][col] != data: - tdLog.exit( - "failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % - (self.sql, row, col, self.queryResult[row][col], data)) + tdLog.exit("%s failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % ( + callerFilename, self.sql, row, col, self.queryResult[row][col], data)) if data is None: tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % @@ -104,22 +116,26 @@ class TDSql: (self.sql, row, col, self.queryResult[row][col], data)) def getData(self, row, col): + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + if row < 0: tdLog.exit( - "failed: sql:%.40s, row:%d is smaller than zero" % - (self.sql, row)) + "%s failed: sql:%.40s, row:%d is smaller than zero" % + (callerFilename, self.sql, row)) if col < 0: tdLog.exit( - "failed: sql:%.40s, col:%d is smaller than zero" % - (self.sql, col)) + "%s failed: sql:%.40s, col:%d is smaller than zero" % + (callerFilename, self.sql, col)) if row >= self.queryRows: tdLog.exit( - "failed: sql:%.40s, row:%d is larger than queryRows:%d" % - (self.sql, row, self.queryRows)) + "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" % + (callerFilename, self.sql, row, self.queryRows)) if col >= self.queryCols: tdLog.exit( - "failed: sql:%.40s, col:%d is larger than queryRows:%d" % - (self.sql, col, self.queryCols)) + "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" % + (callerFilename, self.sql, col, self.queryCols)) return self.queryResult[row][col] def executeTimes(self, sql, times): @@ -137,8 +153,12 @@ class TDSql: def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: - tdLog.exit("failed: sql:%.40s, affectedRows:%d != expect:%d" % - (self.sql, self.affectedRows, expectAffectedRows)) + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + + tdLog.exit("%s failed: sql:%.40s, affectedRows:%d != expect:%d" % ( + callerFilename, self.sql, self.affectedRows, expectAffectedRows)) tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) From 94f4e9bb3875c3d9f649c977600a852d67475f92 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 16 May 2020 15:23:59 +0800 Subject: [PATCH 6/8] add column name length case. --- tests/pytest/table/column_name.py | 91 +++++++------------------------ 1 file changed, 19 insertions(+), 72 deletions(-) diff --git a/tests/pytest/table/column_name.py b/tests/pytest/table/column_name.py index bb1f587a65..aa958fd60c 100644 --- a/tests/pytest/table/column_name.py +++ b/tests/pytest/table/column_name.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- import sys +import string +import random +import subprocess from util.log import * from util.cases import * from util.sql import * @@ -14,34 +17,9 @@ class TDTestCase: def run(self): tdSql.prepare() - # TSIM: system sh/stop_dnodes.sh - # TSIM: - # TSIM: system sh/ip.sh -i 1 -s up - # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 - # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 - # TSIM: system sh/exec.sh -n dnode1 -s start - # TSIM: - # TSIM: sleep 3000 - # TSIM: sql connect - # TSIM: - # TSIM: $i = 0 - # TSIM: $dbPrefix = lm_cm_db - # TSIM: $tbPrefix = lm_cm_tb - # TSIM: $db = $dbPrefix . $i - # TSIM: $tb = $tbPrefix . $i - # TSIM: - # TSIM: print =============== step1 tdLog.info('=============== step1') - # TSIM: sql create database $db - # TSIM: sql use $db - # TSIM: - # TSIM: sql drop table dd -x step0 tdLog.info('drop table dd -x step0') tdSql.error('drop table dd') - # TSIM: return -1 - # TSIM: step0: - # TSIM: - # TSIM: sql create table $tb(ts timestamp, int) -x step1 tdLog.info('create table tb(ts timestamp, int) -x step1') tdSql.error('create table tb(ts timestamp, int)') # TSIM: return -1 @@ -112,37 +90,24 @@ class TDTestCase: tdLog.info('=============== step4') # TSIM: sql create table $tb (ts timestamp, # a0123456789012345678901234567890123456789 int) + getMaxColNum = "grep -w '#define TSDB_COL_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" + boundary = int(subprocess.check_output(getMaxColNum, shell=True)) + tdLog.info("get max column name length is %d" % boundary) + chars = string.ascii_uppercase + string.ascii_lowercase + +# col_name = ''.join(random.choices(chars, k=boundary+1)) +# tdLog.info( +# 'create table tb (ts timestamp, %s int), col_name length is %d' % (col_name, len(col_name))) +# tdSql.error( +# 'create table tb (ts timestamp, %s int)' % col_name) + + col_name = ''.join(random.choices(chars, k=boundary)) tdLog.info( - 'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)') + 'create table tb (ts timestamp, %s int), col_name length is %d' % + (col_name, len(col_name))) tdSql.execute( - 'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)') - # TSIM: sql drop table $tb - tdLog.info('drop table tb') - tdSql.execute('drop table tb') - # TSIM: - # TSIM: sql show tables - tdLog.info('show tables') - tdSql.query('show tables') - # TSIM: if $rows != 0 then - tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step5 - tdLog.info('=============== step5') - # TSIM: sql create table $tb (ts timestamp, a0123456789 int) - tdLog.info('create table tb (ts timestamp, a0123456789 int)') - tdSql.execute('create table tb (ts timestamp, a0123456789 int)') - # TSIM: sql show tables - tdLog.info('show tables') - tdSql.query('show tables') - # TSIM: if $rows != 1 then - tdLog.info('tdSql.checkRow(1)') - tdSql.checkRows(1) - # TSIM: return -1 - # TSIM: endi - # TSIM: + 'create table tb (ts timestamp, %s int)' % col_name) + # TSIM: sql insert into $tb values (now , 1) tdLog.info("insert into tb values (now , 1)") tdSql.execute("insert into tb values (now , 1)") @@ -152,24 +117,6 @@ class TDTestCase: # TSIM: if $rows != 1 then tdLog.info('tdSql.checkRow(1)') tdSql.checkRows(1) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: sql drop database $db - tdLog.info('drop database db') - tdSql.execute('drop database db') - # TSIM: sql show databases - tdLog.info('show databases') - tdSql.query('show databases') - # TSIM: if $rows != 0 then - tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: - # TSIM: - # TSIM: # convert end def stop(self): From bd42e71513aa3490a8fc72f446cfff01f4de1861 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 May 2020 16:15:58 +0800 Subject: [PATCH 7/8] The table name stored in mnode is changed to a variable-length string --- src/inc/taosdef.h | 2 +- src/mnode/inc/mgmtDef.h | 2 +- src/mnode/inc/mgmtSdb.h | 3 +- src/mnode/src/mgmtSdb.c | 137 ++++++++++++++++++++++---------------- src/mnode/src/mgmtTable.c | 127 ++++++++++++++++++++++------------- src/tsdb/src/tsdbMeta.c | 2 +- 6 files changed, 164 insertions(+), 109 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 5dad1bdb53..943fb9e249 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -236,7 +236,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100) #define TSDB_DEFAULT_PAYLOAD_SIZE 1024 // default payload size #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth -#define TSDB_SQLCMD_SIZE 1024 +#define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MAX_VNODES 256 #define TSDB_MIN_VNODES 50 #define TSDB_INVALID_VNODE_NUM 0 diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index f33531583a..894ee87099 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -68,7 +68,7 @@ typedef struct SMnodeObj { // todo use dynamic length string typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char *tableId; int8_t type; } STableObj; diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index a97b1219a9..975206d52e 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -35,7 +35,8 @@ typedef enum { typedef enum { SDB_KEY_STRING, SDB_KEY_INT, - SDB_KEY_AUTO + SDB_KEY_AUTO, + SDB_KEY_VAR_STRING, } ESdbKey; typedef enum { diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index c0d389f305..087c84effd 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -104,6 +104,14 @@ bool sdbIsServing() { return tsSdbObj.status == SDB_STATUS_SERVING; } +static void *sdbGetObjKey(SSdbTable *pTable, void *key) { + if (pTable->keyType == SDB_KEY_VAR_STRING) { + return *(char **)key; + } + + return key; +} + static char *sdbGetActionStr(int32_t action) { switch (action) { case SDB_ACTION_INSERT: @@ -116,20 +124,25 @@ static char *sdbGetActionStr(int32_t action) { return "invalid"; } -static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { +static char *sdbGetKeyStr(SSdbTable *pTable, void *key) { static char str[16]; switch (pTable->keyType) { case SDB_KEY_STRING: - return (char *)row; + case SDB_KEY_VAR_STRING: + return (char *)key; case SDB_KEY_INT: case SDB_KEY_AUTO: - sprintf(str, "%d", *(int32_t *)row); + sprintf(str, "%d", *(int32_t *)key); return str; default: return "invalid"; } } +static char *sdbGetKeyStrFromObj(SSdbTable *pTable, void *key) { + return sdbGetKeyStr(pTable, sdbGetObjKey(pTable, key)); +} + static void *sdbGetTableFromId(int32_t tableId) { return tsSdbObj.tableList[tableId]; } @@ -332,50 +345,48 @@ void sdbCleanUp() { pthread_mutex_destroy(&tsSdbObj.mutex); } -void sdbIncRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); - atomic_add_fetch_32(pRefCount, 1); - if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { - sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), - *pRefCount); - } +void sdbIncRef(void *handle, void *pObj) { + if (pObj == NULL) return; + + SSdbTable *pTable = handle; + int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); + atomic_add_fetch_32(pRefCount, 1); + if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { + sdbTrace("table:%s, add ref to record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); } } -void sdbDecRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); - int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { - sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), - *pRefCount); - } - int8_t *updateEnd = pRow + pTable->refCountPos - 1; - if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, - sdbGetkeyStr(pTable, pRow), *pRefCount); - SSdbOper oper = {.pObj = pRow}; - (*pTable->destroyFp)(&oper); - } +void sdbDecRef(void *handle, void *pObj) { + if (pObj == NULL) return; + + SSdbTable *pTable = handle; + int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); + int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); + if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { + sdbTrace("table:%s, def ref of record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); + } + + int8_t *updateEnd = pObj + pTable->refCountPos - 1; + if (refCount <= 0 && *updateEnd) { + sdbTrace("table:%s, record:%s:%d is destroyed", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); + SSdbOper oper = {.pObj = pObj}; + (*pTable->destroyFp)(&oper); } } -static SSdbRow *sdbGetRowMeta(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SSdbRow * pMeta; - - if (handle == NULL) return NULL; +static SSdbRow *sdbGetRowMeta(SSdbTable *pTable, void *key) { + if (pTable == NULL) return NULL; int32_t keySize = sizeof(int32_t); - if (pTable->keyType == SDB_KEY_STRING) { + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { keySize = strlen((char *)key); } - pMeta = taosHashGet(pTable->iHandle, key, keySize); + + return taosHashGet(pTable->iHandle, key, keySize); +} - return pMeta; +static SSdbRow *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) { + return sdbGetRowMeta(pTable, sdbGetObjKey(pTable, key)); } void *sdbGetRow(void *handle, void *key) { @@ -387,7 +398,7 @@ void *sdbGetRow(void *handle, void *key) { pthread_mutex_lock(&pTable->mutex); int32_t keySize = sizeof(int32_t); - if (pTable->keyType == SDB_KEY_STRING) { + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { keySize = strlen((char *)key); } pMeta = taosHashGet(pTable->iHandle, key, keySize); @@ -400,6 +411,10 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } +static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) { + return sdbGetRow(pTable, sdbGetObjKey(pTable, key)); +} + static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { SSdbRow rowMeta; rowMeta.rowSize = pOper->rowSize; @@ -407,11 +422,14 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { pthread_mutex_lock(&pTable->mutex); + void * key = sdbGetObjKey(pTable, pOper->pObj); int32_t keySize = sizeof(int32_t); - if (pTable->keyType == SDB_KEY_STRING) { - keySize = strlen((char *)pOper->pObj); + + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { + keySize = strlen((char *)key); } - taosHashPut(pTable->iHandle, pOper->pObj, keySize, &rowMeta, sizeof(SSdbRow)); + + taosHashPut(pTable->iHandle, key, keySize, &rowMeta, sizeof(SSdbRow)); sdbIncRef(pTable, pOper->pObj); pTable->numOfRows++; @@ -425,7 +443,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); (*pTable->insertFp)(pOper); return TSDB_CODE_SUCCESS; @@ -436,17 +454,20 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { pthread_mutex_lock(&pTable->mutex); + void * key = sdbGetObjKey(pTable, pOper->pObj); int32_t keySize = sizeof(int32_t); - if (pTable->keyType == SDB_KEY_STRING) { - keySize = strlen((char *)pOper->pObj); + + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { + keySize = strlen((char *)key); } - taosHashRemove(pTable->iHandle, pOper->pObj, keySize); + + taosHashRemove(pTable->iHandle, key, keySize); pTable->numOfRows--; pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; *updateEnd = 1; @@ -457,7 +478,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; @@ -488,7 +509,7 @@ static int sdbWrite(void *param, void *data, int type) { } else if (pHead->version != tsSdbObj.version + 1) { pthread_mutex_unlock(&tsSdbObj.mutex); sdbError("table:%s, failed to restore %s record:%s from wal, version:%" PRId64 " too large, sdb version:%" PRId64, - pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, + pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, tsSdbObj.version); return TSDB_CODE_OTHERS; } else { @@ -540,8 +561,8 @@ int32_t sdbInsertRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - if (sdbGetRow(pTable, pOper->pObj)) { - sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + if (sdbGetRowFromObj(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj)); sdbDecRef(pTable, pOper->pObj); return TSDB_CODE_ALREADY_THERE; } @@ -580,7 +601,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pMeta == NULL) { sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; @@ -590,25 +611,27 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { assert(pMetaRow != NULL); if (pOper->type == SDB_OPER_GLOBAL) { - int32_t rowSize = 0; + void * key = sdbGetObjKey(pTable, pOper->pObj); + int32_t keySize = 0; switch (pTable->keyType) { case SDB_KEY_STRING: - rowSize = strlen((char *)pOper->pObj) + 1; + case SDB_KEY_VAR_STRING: + keySize = strlen((char *)key) + 1; break; case SDB_KEY_INT: case SDB_KEY_AUTO: - rowSize = sizeof(uint64_t); + keySize = sizeof(uint32_t); break; default: return -1; } - int32_t size = sizeof(SWalHead) + rowSize; + int32_t size = sizeof(SWalHead) + keySize; SWalHead *pHead = taosAllocateQitem(size); pHead->version = 0; - pHead->len = rowSize; + pHead->len = keySize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; - memcpy(pHead->cont, pOper->pObj, rowSize); + memcpy(pHead->cont, key, keySize); int32_t code = sdbWrite(pOper, pHead, pHead->msgType); taosFreeQitem(pHead); @@ -622,7 +645,7 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pMeta == NULL) { sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 3e3b16fa9d..28b3c6e3dd 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -84,6 +84,7 @@ static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); static void mgmtDestroyChildTable(SChildTableObj *pTable) { + tfree(pTable->info.tableId); tfree(pTable->schema); tfree(pTable->sql); tfree(pTable); @@ -180,6 +181,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) { SChildTableObj *pNew = pOper->pObj; SChildTableObj *pTable = mgmtGetChildTable(pNew->info.tableId); if (pTable != pNew) { + void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; void *oldSchema = pTable->schema; memcpy(pTable, pNew, pOper->rowSize); @@ -188,6 +190,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) { free(pNew); free(oldSql); free(oldSchema); + free(oldTableId); } mgmtDecTableRef(pTable); @@ -195,51 +198,66 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) { } static int32_t mgmtChildTableActionEncode(SSdbOper *pOper) { - const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16); SChildTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); - if (pTable->info.type == TSDB_CHILD_TABLE) { - memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - pOper->rowSize = tsChildTableUpdateSize; - } else { + int32_t len = strlen(pTable->info.tableId); + if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID; + + memcpy(pOper->rowData, pTable->info.tableId, len); + memset(pOper->rowData + len, 0, 1); + len++; + + memcpy(pOper->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + len += tsChildTableUpdateSize; + + if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (maxRowSize < tsChildTableUpdateSize + schemaSize) { - return TSDB_CODE_INVALID_MSG_LEN; + memcpy(pOper->rowData + len, pTable->schema, schemaSize); + len += schemaSize; + + if (pTable->sqlLen != 0) { + memcpy(pOper->rowData + len, pTable->sql, pTable->sqlLen); + len += pTable->sqlLen; } - memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - memcpy(pOper->rowData + tsChildTableUpdateSize, pTable->schema, schemaSize); - memcpy(pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - pOper->rowSize = tsChildTableUpdateSize + schemaSize + pTable->sqlLen; } + pOper->rowSize = len; + return TSDB_CODE_SUCCESS; } static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); - if (pTable == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; - } + if (pTable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - memcpy(pTable, pOper->rowData, tsChildTableUpdateSize); + int32_t len = strlen(pOper->rowData); + if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID; + pTable->info.tableId = strdup(pOper->rowData); + len++; + + memcpy((char*)pTable + sizeof(char *), pOper->rowData + len, tsChildTableUpdateSize); + len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { mgmtDestroyChildTable(pTable); - return TSDB_CODE_SERV_OUT_OF_MEMORY; + return TSDB_CODE_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pOper->rowData + tsChildTableUpdateSize, schemaSize); + memcpy(pTable->schema, pOper->rowData + len, schemaSize); + len += schemaSize; - pTable->sql = (char *)malloc(pTable->sqlLen); - if (pTable->sql == NULL) { - mgmtDestroyChildTable(pTable); - return TSDB_CODE_SERV_OUT_OF_MEMORY; + if (pTable->sqlLen != 0) { + pTable->sql = malloc(pTable->sqlLen); + if (pTable->sql == NULL) { + mgmtDestroyChildTable(pTable); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + memcpy(pTable->sql, pOper->rowData + len, pTable->sqlLen); } - memcpy(pTable->sql, pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sqlLen); } pOper->pObj = pTable; @@ -311,15 +329,15 @@ static int32_t mgmtChildTableActionRestored() { static int32_t mgmtInitChildTables() { SChildTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_CTABLE, .tableName = "ctables", .hashSessions = tsMaxTables, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16), + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_STRING, + .keyType = SDB_KEY_VAR_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, .updateFp = mgmtChildTableActionUpdate, @@ -372,6 +390,7 @@ static void mgmtDestroySuperTable(SSuperTableObj *pStable) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; } + tfree(pStable->info.tableId); tfree(pStable->schema); tfree(pStable); } @@ -408,11 +427,13 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) { SSuperTableObj *pNew = pOper->pObj; SSuperTableObj *pTable = mgmtGetSuperTable(pNew->info.tableId); if (pTable != pNew) { + void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; memcpy(pTable, pNew, pOper->rowSize); pTable->schema = pNew->schema; free(pNew->vgHash); free(pNew); + free(oldTableId); free(oldSchema); } mgmtDecTableRef(pTable); @@ -420,40 +441,50 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) { } static int32_t mgmtSuperTableActionEncode(SSdbOper *pOper) { - const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16); - SSuperTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); + int32_t len = strlen(pStable->info.tableId); + if (len > TSDB_TABLE_ID_LEN) len = TSDB_CODE_INVALID_TABLE_ID; + + memcpy(pOper->rowData, pStable->info.tableId, len); + memset(pOper->rowData + len, 0, 1); + len++; + + memcpy(pOper->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + len += tsSuperTableUpdateSize; + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + memcpy(pOper->rowData + len, pStable->schema, schemaSize); + len += schemaSize; - if (maxRowSize < tsSuperTableUpdateSize + schemaSize) { - return TSDB_CODE_INVALID_MSG_LEN; - } - - memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); - memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); - pOper->rowSize = tsSuperTableUpdateSize + schemaSize; + pOper->rowSize = len; return TSDB_CODE_SUCCESS; } static int32_t mgmtSuperTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); - SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); + int32_t len = strlen(pOper->rowData); + if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID; + pStable->info.tableId = strdup(pOper->rowData); + len++; + + memcpy((char*)pStable + sizeof(char *), pOper->rowData + len, tsSuperTableUpdateSize); + len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); pStable->schema = malloc(schemaSize); if (pStable->schema == NULL) { mgmtDestroySuperTable(pStable); - return -1; + return TSDB_CODE_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + memcpy(pStable->schema, pOper->rowData + len, schemaSize); + pOper->pObj = pStable; return TSDB_CODE_SUCCESS; @@ -465,15 +496,15 @@ static int32_t mgmtSuperTableActionRestored() { static int32_t mgmtInitSuperTables() { SSuperTableObj tObj; - tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_MAX_SUPER_TABLES, - .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16), + .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_STRING, + .keyType = SDB_KEY_VAR_STRING, .insertFp = mgmtSuperTableActionInsert, .deleteFp = mgmtSuperTableActionDelete, .updateFp = mgmtSuperTableActionUpdate, @@ -720,14 +751,14 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; - SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); + SSuperTableObj *pStable = calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { mError("table:%s, failed to create, no enough memory", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); return; } - strcpy(pStable->info.tableId, pCreate->tableId); + pStable->info.tableId = strdup(pCreate->tableId); pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); @@ -1358,7 +1389,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb } static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - SChildTableObj *pTable = (SChildTableObj *) calloc(1, sizeof(SChildTableObj)); + SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; @@ -1371,10 +1402,10 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj pTable->info.type = TSDB_NORMAL_TABLE; } - strcpy(pTable->info.tableId, pCreate->tableId); - pTable->createdTime = taosGetTimestampMs(); - pTable->sid = tid; - pTable->vgId = pVgroup->vgId; + pTable->info.tableId = strdup(pCreate->tableId); + pTable->createdTime = taosGetTimestampMs(); + pTable->sid = tid; + pTable->vgId = pVgroup->vgId; if (pTable->info.type == TSDB_CHILD_TABLE) { char *pTagData = (char *) pCreate->schema; // it is a tag key diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index aaea50267d..5ecad18443 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -80,7 +80,7 @@ STable *tsdbDecodeTable(void *cont, int contLen) { T_READ_MEMBER(ptr, int8_t, pTable->type); int len = *(int *)ptr; ptr = (char *)ptr + sizeof(int); - pTable->name = calloc(1, len + VARSTR_HEADER_SIZE); + pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1); if (pTable->name == NULL) return NULL; varDataSetLen(pTable->name, len); From 27f5a1eaf62008b29da8bd58229d78b51cac3f15 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 May 2020 08:56:14 +0000 Subject: [PATCH 8/8] add more debug info --- src/vnode/src/vnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 0f92be0967..6dabc98ae8 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -429,7 +429,7 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { SVnodeObj *pVnode = ahandle; - vTrace("vgId:%d, data file is synced", pVnode->vgId); + vTrace("vgId:%d, data file is synced, fversion:%" PRId64 "", pVnode->vgId, fversion); pVnode->fversion = fversion; pVnode->version = fversion;