Merge branch 'develop' into feature/mwrite

src/client/inc/tscUtil.h

@@ -43,7 +43,7 @@ extern "C" {
 
 typedef struct SParsedColElem {
   int16_t colIndex;
-  int16_t offset;
+  uint16_t offset;
 } SParsedColElem;
 
 typedef struct SParsedDataColInfo {

src/client/inc/tsclient.h

@@ -49,7 +49,7 @@ typedef struct STableComInfo {
   uint8_t numOfTags;
   uint8_t precision;
   int16_t numOfColumns;
-  int16_t rowSize;
+  int32_t rowSize;
 } STableComInfo;
 
 typedef struct STableMeta {

src/inc/taosdef.h

@@ -193,20 +193,20 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_ACCT_LEN             TSDB_UNI_LEN
 #define TSDB_PASSWORD_LEN         TSDB_UNI_LEN
 
-#define TSDB_MAX_COLUMNS          256
+#define TSDB_MAX_COLUMNS          1024
 #define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
 
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       192
 #define TSDB_DB_NAME_LEN          32
 #define TSDB_COL_NAME_LEN         64
-#define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 16
+#define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
 #define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 6mb
 
-#define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 16
+#define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 64
-#define TSDB_MAX_TAGS_LEN         512
+#define TSDB_MAX_TAGS_LEN         65536
-#define TSDB_MAX_TAGS             32
+#define TSDB_MAX_TAGS             128
 
 #define TSDB_AUTH_LEN             16
 #define TSDB_KEY_LEN              16

src/rpc/src/rpcMain.c

@@ -361,9 +361,10 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
   // connection type is application specific. 
   // for TDengine, all the query, show commands shall have TCP connection
   char type = pMsg->msgType;
-  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH ||
+  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE
-      type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META ||
+    || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP
-      type == TSDB_MSG_TYPE_CM_SHOW )
+    || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META
+    || type == TSDB_MSG_TYPE_CM_SHOW )
     pContext->connType = RPC_CONN_TCPC;
   
   rpcSendReqToServer(pRpc, pContext);
@@ -1109,6 +1110,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
           htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
+  tTrace("connection type is: %d", pConn->connType);
   writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle);
 
   if (writtenLen != msgLen) {

src/vnode/src/vnodeMain.c

@@ -429,7 +429,7 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
 
 static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) {
   SVnodeObj *pVnode = ahandle;
-  vTrace("vgId:%d, data file is synced", pVnode->vgId);
+  vTrace("vgId:%d, data file is synced, fversion:%" PRId64 "", pVnode->vgId, fversion);
 
   pVnode->fversion = fversion;
   pVnode->version = fversion;

tests/pytest/table/boundary.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+import random
+import string
+import subprocess
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init( self, conn ):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+
+    def getLimitFromSourceCode( self, name ):
+        cmd = "grep -w '#define %s' ../../src/inc/taosdef.h|awk '{print $3}'" % name
+        return int(subprocess.check_output(cmd, shell=True))
+
+
+    def generateString( self, length ):
+        chars = string.ascii_uppercase + string.ascii_lowercase
+        v = ""
+        for i in range( length ):
+            v += random.choice( chars )
+        return v
+
+
+    def checkTagBoundaries( self ):
+        tdLog.debug( "checking tag boundaries" )
+        tdSql.prepare()
+
+        maxTags = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS' )
+        totalTagsLen = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS_LEN' )
+        tdLog.notice( "max tags is %d" % maxTags )
+        tdLog.notice( "max total tag length is %d" % totalTagsLen )
+
+        # for binary tags, 2 bytes are used for length
+        tagLen = (totalTagsLen - maxTags * 2) // maxTags
+        firstTagLen = totalTagsLen - 2 * maxTags - tagLen * (maxTags - 1)
+
+        sql = "create table cars(ts timestamp, f int) tags(t0 binary(%d)" % firstTagLen
+        for i in range( 1, maxTags ):
+            sql += ", t%d binary(%d)" % (i, tagLen)
+        sql += ");"
+
+        tdLog.debug( "creating super table: " + sql )
+        tdSql.execute( sql )
+        tdSql.query( 'show stables' )
+        tdSql.checkRows( 1 )
+
+        for i in range( 10 ):
+            sql = "create table car%d using cars tags('%d'" % (i, i)
+            sql += ", '0'" * (maxTags - 1) + ");"
+            tdLog.debug( "creating table: " + sql )
+            tdSql.execute( sql )
+
+            sql = "insert into car%d values(now, 0);" % i
+            tdLog.debug( "inserting data: " + sql )
+            tdSql.execute( sql )
+
+        tdSql.query( 'show tables' )
+        tdLog.info( 'tdSql.checkRow(10)' )
+        tdSql.checkRows( 10 )
+
+        tdSql.query( 'select * from cars;' )
+        tdSql.checkRows( 10 )
+
+
+    def checkColumnBoundaries( self ):
+        tdLog.debug( "checking column boundaries" )
+        tdSql.prepare()
+
+        # one column is for timestamp
+        maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1
+
+        sql = "create table cars (ts timestamp"
+        for i in range( maxCols ):
+            sql += ", c%d int" % i
+        sql += ");"
+        tdSql.execute( sql )
+        tdSql.query( 'show tables' )
+        tdSql.checkRows( 1 )
+
+        sql = "insert into cars values (now"
+        for i in range( maxCols ):
+            sql += ", %d" % i
+        sql += ");"
+        tdSql.execute( sql )
+        tdSql.query( 'select * from cars' )
+        tdSql.checkRows( 1 )
+
+
+    def checkTableNameBoundaries( self ):
+        tdLog.debug( "checking table name boundaries" )
+        tdSql.prepare()
+
+        maxTableNameLen = self.getLimitFromSourceCode( 'TSDB_TABLE_NAME_LEN' )
+        tdLog.notice( "table name max length is %d" % maxTableNameLen )
+
+        name = self.generateString( maxTableNameLen - 1)
+        tdLog.info( "table name is '%s'" % name )
+
+        tdSql.execute( "create table %s (ts timestamp, value int)" % name )
+        tdSql.execute( "insert into %s values(now, 0)" % name )
+
+        tdSql.query( 'show tables' )
+        tdSql.checkRows( 1 )
+
+        tdSql.query( 'select * from %s' % name )
+        tdSql.checkRows( 1 )
+
+
+    def checkRowBoundaries( self ):
+        tdLog.debug( "checking row boundaries" )
+        tdSql.prepare()
+
+        # 8 bytes for timestamp
+        maxRowSize = 65536 - 8
+        maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1
+
+        # for binary cols, 2 bytes are used for length
+        colLen = (maxRowSize - maxCols * 2) // maxCols
+        firstColLen = maxRowSize - 2 * maxCols - colLen * (maxCols - 1)
+
+        sql = "create table cars (ts timestamp, c0 binary(%d)" % firstColLen
+        for i in range( 1, maxCols ):
+            sql += ", c%d binary(%d)" % (i, colLen)
+        sql += ");"
+        tdSql.execute( sql )
+        tdSql.query( 'show tables' )
+        tdSql.checkRows( 1 )
+
+        col = self.generateString( firstColLen )
+        sql = "insert into cars values (now, '%s'" % col
+        col = self.generateString( colLen )
+        for i in range( 1, maxCols ):
+            sql += ", '%s'" % col 
+        sql += ");"
+        tdLog.info( sql );
+        tdSql.execute( sql )
+        tdSql.query( "select * from cars" )
+        tdSql.checkRows( 1 )
+
+
+    def run(self):
+        self.checkTagBoundaries()
+        self.checkColumnBoundaries()
+        self.checkTableNameBoundaries()
+        self.checkRowBoundaries()
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/table/column_name.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 
 import sys
+import string
+import random
+import subprocess
 from util.log import *
 from util.cases import *
 from util.sql import *
@@ -14,34 +17,9 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        # TSIM: system sh/stop_dnodes.sh
-        # TSIM:
-        # TSIM: system sh/ip.sh -i 1 -s up
-        # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
-        # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0
-        # TSIM: system sh/exec.sh -n dnode1 -s start
-        # TSIM:
-        # TSIM: sleep 3000
-        # TSIM: sql connect
-        # TSIM:
-        # TSIM: $i = 0
-        # TSIM: $dbPrefix = lm_cm_db
-        # TSIM: $tbPrefix = lm_cm_tb
-        # TSIM: $db = $dbPrefix . $i
-        # TSIM: $tb = $tbPrefix . $i
-        # TSIM:
-        # TSIM: print =============== step1
         tdLog.info('=============== step1')
-        # TSIM: sql create database $db
-        # TSIM: sql use $db
-        # TSIM:
-        # TSIM: sql drop table dd -x step0
         tdLog.info('drop table dd -x step0')
         tdSql.error('drop table dd')
-        # TSIM: return -1
-        # TSIM: step0:
-        # TSIM:
-        # TSIM: sql create table $tb(ts timestamp, int) -x step1
         tdLog.info('create table tb(ts timestamp, int) -x step1')
         tdSql.error('create table tb(ts timestamp, int)')
         # TSIM: return -1
@@ -112,37 +90,24 @@ class TDTestCase:
         tdLog.info('=============== step4')
         # TSIM: sql create table $tb (ts timestamp,
         # a0123456789012345678901234567890123456789 int)
+        getMaxColNum = "grep -w '#define TSDB_COL_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'"
+        boundary = int(subprocess.check_output(getMaxColNum, shell=True))
+        tdLog.info("get max column name length is %d" % boundary)
+        chars = string.ascii_uppercase + string.ascii_lowercase
+
+#        col_name = ''.join(random.choices(chars, k=boundary+1))
+#        tdLog.info(
+#            'create table tb (ts timestamp, %s int), col_name length is %d' % (col_name, len(col_name)))
+#        tdSql.error(
+#            'create table tb (ts timestamp, %s int)' % col_name)
+
+        col_name = ''.join(random.choices(chars, k=boundary))
         tdLog.info(
-            'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
+            'create table tb (ts timestamp, %s int), col_name length is %d' %
+            (col_name, len(col_name)))
         tdSql.execute(
-            'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
+            'create table tb (ts timestamp, %s int)' % col_name)
-        # TSIM: sql drop table $tb
+
-        tdLog.info('drop table tb')
-        tdSql.execute('drop table tb')
-        # TSIM:
-        # TSIM: sql show tables
-        tdLog.info('show tables')
-        tdSql.query('show tables')
-        # TSIM: if $rows != 0 then
-        tdLog.info('tdSql.checkRow(0)')
-        tdSql.checkRows(0)
-        # TSIM: return -1
-        # TSIM: endi
-        # TSIM:
-        # TSIM: print =============== step5
-        tdLog.info('=============== step5')
-        # TSIM: sql create table $tb (ts timestamp, a0123456789 int)
-        tdLog.info('create table tb (ts timestamp, a0123456789 int)')
-        tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
-        # TSIM: sql show tables
-        tdLog.info('show tables')
-        tdSql.query('show tables')
-        # TSIM: if $rows != 1 then
-        tdLog.info('tdSql.checkRow(1)')
-        tdSql.checkRows(1)
-        # TSIM: return -1
-        # TSIM: endi
-        # TSIM:
         # TSIM: sql insert into $tb values (now , 1)
         tdLog.info("insert into tb values (now , 1)")
         tdSql.execute("insert into tb values (now , 1)")
@@ -152,24 +117,6 @@ class TDTestCase:
         # TSIM: if $rows != 1 then
         tdLog.info('tdSql.checkRow(1)')
         tdSql.checkRows(1)
-        # TSIM: return -1
-        # TSIM: endi
-        # TSIM:
-        # TSIM: sql drop database $db
-        tdLog.info('drop database db')
-        tdSql.execute('drop database db')
-        # TSIM: sql show databases
-        tdLog.info('show databases')
-        tdSql.query('show databases')
-        # TSIM: if $rows != 0 then
-        tdLog.info('tdSql.checkRow(0)')
-        tdSql.checkRows(0)
-        # TSIM: return -1
-        # TSIM: endi
-        # TSIM:
-        # TSIM:
-        # TSIM:
-        # TSIM:
 # convert end
 
     def stop(self):

tests/pytest/table/column_num.py

@@ -76,7 +76,7 @@ class TDTestCase:
         tdSql.checkRows(2)
 
         data = "now"
-        for x in range(0, boundary-1):
+        for x in range(0, boundary - 1):
             data = data + ", %d" % x
         tdLog.info("insert into tb1 values (%s)" % data)
         tdSql.execute("insert into tb1 values (%s)" % data)

tests/pytest/table/tablename-boundary.py

@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import sys
-import string
-import random
-import subprocess
-from util.log import *
-from util.cases import *
-from util.sql import *
-
-
-class TDTestCase:
-    def init(self, conn):
-        tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
-
-    def run(self):
-        tdSql.prepare()
-
-        getTableNameLen = "grep -w '#define TSDB_TABLE_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'"
-        tableNameMaxLen = int(
-            subprocess.check_output(
-                getTableNameLen, shell=True))
-        tdLog.notice("table name max length is %d" % tableNameMaxLen)
-        chars = string.ascii_uppercase + string.ascii_lowercase
-        tb_name = ''.join(random.choices(chars, k=tableNameMaxLen))
-        tdLog.info('tb_name length %d' % len(tb_name))
-        tdLog.info('create table %s (ts timestamp, value int)' % tb_name)
-        tdSql.error(
-            'create table %s (ts timestamp, speed binary(4089))' %
-            tb_name)
-
-        tb_name = ''.join(random.choices(chars, k=191))
-        tdLog.info('tb_name length %d' % len(tb_name))
-        tdLog.info('create table %s (ts timestamp, value int)' % tb_name)
-        tdSql.execute(
-            'create table %s (ts timestamp, speed binary(4089))' %
-            tb_name)
-
-    def stop(self):
-        tdSql.close()
-        tdLog.success("%s successfully executed" % __file__)
-
-
-tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())

tests/pytest/util/cases.py

@@ -71,7 +71,7 @@ class TDCases:
                     case.run()
                 except Exception as e:
                     tdLog.notice(repr(e))
-                    tdLog.exit("%s failed: %s" % (__file__, fileName))
+                    tdLog.exit("%s failed" % (fileName))
                 case.stop()
                 runNum += 1
                 continue

tests/pytest/util/sql.py

@@ -15,6 +15,7 @@ import sys
 import os
 import time
 import datetime
+import inspect
 from util.log import *
 
 
@@ -44,7 +45,12 @@ class TDSql:
         except BaseException:
             expectErrNotOccured = False
         if expectErrNotOccured:
-            tdLog.exit("failed: sql:%.40s, expect error not occured" % (sql))
+            frame = inspect.stack()[1]
+            callerModule = inspect.getmodule(frame[0])
+            callerFilename = callerModule.__file__
+            tdLog.exit(
+                "%s failed: sql:%.40s, expect error not occured" %
+                (callerFilename, sql))
         else:
             tdLog.info("sql:%.40s, expect error occured" % (sql))
 
@@ -62,33 +68,39 @@ class TDSql:
 
     def checkRows(self, expectRows):
         if self.queryRows != expectRows:
+            frame = inspect.stack()[1]
+            callerModule = inspect.getmodule(frame[0])
+            callerFilename = callerModule.__file__
             tdLog.exit(
-                "failed: sql:%.40s, queryRows:%d != expect:%d" %
+                "%s failed: sql:%.40s, queryRows:%d != expect:%d" %
-                (self.sql, self.queryRows, expectRows))
+                (callerFilename, self.sql, self.queryRows, expectRows))
         tdLog.info("sql:%.40s, queryRows:%d == expect:%d" %
                    (self.sql, self.queryRows, expectRows))
 
     def checkData(self, row, col, data):
+        frame = inspect.stack()[1]
+        callerModule = inspect.getmodule(frame[0])
+        callerFilename = callerModule.__file__
+
         if row < 0:
             tdLog.exit(
-                "failed: sql:%.40s, row:%d is smaller than zero" %
+                "%s failed: sql:%.40s, row:%d is smaller than zero" %
-                (self.sql, row))
+                (callerFilename, self.sql, row))
         if col < 0:
             tdLog.exit(
-                "failed: sql:%.40s, col:%d is smaller than zero" %
+                "%s failed: sql:%.40s, col:%d is smaller than zero" %
-                (self.sql, col))
+                (callerFilename, self.sql, col))
         if row >= self.queryRows:
             tdLog.exit(
-                "failed: sql:%.40s, row:%d is larger than queryRows:%d" %
+                "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" %
-                (self.sql, row, self.queryRows))
+                (callerFilename, self.sql, row, self.queryRows))
         if col >= self.queryCols:
             tdLog.exit(
-                "failed: sql:%.40s, col:%d is larger than queryRows:%d" %
+                "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" %
-                (self.sql, col, self.queryCols))
+                (callerFilename, self.sql, col, self.queryCols))
         if self.queryResult[row][col] != data:
-            tdLog.exit(
+            tdLog.exit("%s failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % (
-                "failed: sql:%.40s row:%d col:%d data:%s != expect:%s" %
+                callerFilename, self.sql, row, col, self.queryResult[row][col], data))
-                (self.sql, row, col, self.queryResult[row][col], data))
 
         if data is None:
             tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" %
@@ -104,22 +116,26 @@ class TDSql:
                        (self.sql, row, col, self.queryResult[row][col], data))
 
     def getData(self, row, col):
+        frame = inspect.stack()[1]
+        callerModule = inspect.getmodule(frame[0])
+        callerFilename = callerModule.__file__
+
         if row < 0:
             tdLog.exit(
-                "failed: sql:%.40s, row:%d is smaller than zero" %
+                "%s failed: sql:%.40s, row:%d is smaller than zero" %
-                (self.sql, row))
+                (callerFilename, self.sql, row))
         if col < 0:
             tdLog.exit(
-                "failed: sql:%.40s, col:%d is smaller than zero" %
+                "%s failed: sql:%.40s, col:%d is smaller than zero" %
-                (self.sql, col))
+                (callerFilename, self.sql, col))
         if row >= self.queryRows:
             tdLog.exit(
-                "failed: sql:%.40s, row:%d is larger than queryRows:%d" %
+                "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" %
-                (self.sql, row, self.queryRows))
+                (callerFilename, self.sql, row, self.queryRows))
         if col >= self.queryCols:
             tdLog.exit(
-                "failed: sql:%.40s, col:%d is larger than queryRows:%d" %
+                "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" %
-                (self.sql, col, self.queryCols))
+                (callerFilename, self.sql, col, self.queryCols))
         return self.queryResult[row][col]
 
     def executeTimes(self, sql, times):
@@ -137,8 +153,12 @@ class TDSql:
 
     def checkAffectedRows(self, expectAffectedRows):
         if self.affectedRows != expectAffectedRows:
-            tdLog.exit("failed: sql:%.40s, affectedRows:%d != expect:%d" %
+            frame = inspect.stack()[1]
-                       (self.sql, self.affectedRows, expectAffectedRows))
+            callerModule = inspect.getmodule(frame[0])
+            callerFilename = callerModule.__file__
+
+            tdLog.exit("%s failed: sql:%.40s, affectedRows:%d != expect:%d" % (
+                callerFilename, self.sql, self.affectedRows, expectAffectedRows))
         tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" %
                    (self.sql, self.affectedRows, expectAffectedRows))