Merge branch 'develop' into test/TD-4463

src/client/src/tscUtil.c

@@ -1119,6 +1119,8 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
 
     SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols);
 
+    pOutput->precision = pSqlObjList[0]->res.precision;
+    
     SSchema* schema = NULL;
     if (px->numOfTables > 1) {
       SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java

@@ -243,15 +243,93 @@ public class TSDBPreparedStatementTest {
             s.setNString(1, s2, 4);                   
 
             random = 10 + r.nextInt(5);
-            ArrayList<String> s5 = new ArrayList<String>();
+            ArrayList<String> s3 = new ArrayList<String>();
             for(int i = 0; i < numOfRows; i++) {
                 if(i % random == 0) {
-                    s5.add(null);
+                    s3.add(null);
                 }else{
-                    s5.add("test" + i % 10);
+                    s3.add("test" + i % 10);
                 }
             }
-            s.setString(2, s5, 10);        
+            s.setString(2, s3, 10);        
+
+            s.columnDataAddBatch();
+            s.columnDataExecuteBatch();
+            s.columnDataCloseBatch();
+
+            String sql = "select * from weather_test";
+            PreparedStatement statement = conn.prepareStatement(sql);
+            ResultSet rs = statement.executeQuery();
+            int rows = 0;
+            while(rs.next()) {
+                rows++;
+            }
+            Assert.assertEquals(numOfRows, rows);            
+        }
+    }
+
+    @Test
+    public void bindDataWithSingleTagTest() throws SQLException { 
+        Statement stmt = conn.createStatement();                
+
+        String types[] = new String[] {"tinyint", "smallint", "int", "bigint", "bool", "float", "double", "binary(10)", "nchar(10)"};                
+
+        for (String type : types) {
+            stmt.execute("drop table if exists weather_test");
+            stmt.execute("create table weather_test(ts timestamp, f1 nchar(10), f2 binary(10)) tags (t " + type + ")");
+
+            int numOfRows = 1;
+    
+            TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags(?) values(?, ?, ?)");
+            Random r = new Random();
+            s.setTableName("w1");
+            
+            switch(type) {
+                case "tinyint":
+                case "smallint":
+                case "int":
+                case "bigint":
+                    s.setTagInt(0, 1);
+                    break;
+                case "float":
+                    s.setTagFloat(0, 1.23f);
+                    break;
+                case "double":
+                    s.setTagDouble(0, 3.14159265);
+                    break;
+                case "bool":
+                    s.setTagBoolean(0, true);
+                    break;
+                case "binary(10)":
+                    s.setTagString(0, "test");
+                    break;
+                case "nchar(10)":
+                    s.setTagNString(0, "test");
+                    break;
+                default:
+                    break;
+            }
+            
+            
+            ArrayList<Long> ts = new ArrayList<Long>();
+            for(int i = 0; i < numOfRows; i++) {
+                ts.add(System.currentTimeMillis() + i);
+            }        
+            s.setTimestamp(0, ts);
+    
+            int random = 10 + r.nextInt(5);
+            ArrayList<String> s2 = new ArrayList<String>();        
+            for(int i = 0; i < numOfRows; i++) {            
+                s2.add("分支" + i % 4);            
+            }        
+            s.setNString(1, s2, 10);
+            
+            random = 10 + r.nextInt(5);
+            ArrayList<String> s3 = new ArrayList<String>();        
+            for(int i = 0; i < numOfRows; i++) {            
+                s3.add("test" + i % 4);            
+            }        
+            s.setString(2, s3, 10);
     
             s.columnDataAddBatch();
             s.columnDataExecuteBatch();
@@ -269,6 +347,47 @@ public class TSDBPreparedStatementTest {
     }
 
 
+    @Test
+    public void bindDataWithMultipleTagsTest() throws SQLException { 
+        Statement stmt = conn.createStatement();                
+        
+        stmt.execute("drop table if exists weather_test");
+        stmt.execute("create table weather_test(ts timestamp, f1 nchar(10), f2 binary(10)) tags (t1 int, t2 binary(10))");
+
+        int numOfRows = 1;
+
+        TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags(?,?) (ts, f2) values(?, ?)");        
+        s.setTableName("w2");  
+        s.setTagInt(0, 1);
+        s.setTagString(1, "test");
+        
+        
+        ArrayList<Long> ts = new ArrayList<Long>();
+        for(int i = 0; i < numOfRows; i++) {
+            ts.add(System.currentTimeMillis() + i);
+        }        
+        s.setTimestamp(0, ts);
+        
+        ArrayList<String> s2 = new ArrayList<String>();        
+        for(int i = 0; i < numOfRows; i++) {            
+            s2.add("test" + i % 4);
+        }
+        s.setString(1, s2, 10);
+    
+        s.columnDataAddBatch();
+        s.columnDataExecuteBatch();
+        s.columnDataCloseBatch();
+
+        String sql = "select * from weather_test";
+        PreparedStatement statement = conn.prepareStatement(sql);
+        ResultSet rs = statement.executeQuery();
+        int rows = 0;
+        while(rs.next()) {
+            rows++;
+        }
+        Assert.assertEquals(numOfRows, rows);
+        
+    }
 
     @Test
     public void setBoolean() throws SQLException {

tests/pytest/manualTest/manual_alter_block.py

@@ -0,0 +1,82 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+
+##TODO: auto test version is currently unsupported, need to come up with 
+#       an auto test version in the future
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+        
+    def run(self):
+        tdSql.prepare()
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"  
+
+        #alter cache block to 3, then check alter
+        tdSql.execute('alter database db blocks 3')
+        tdSql.query('show databases')
+        tdSql.checkData(0,9,3)
+
+        #run taosdemo to occupy all cache, need to manually check memory consumption
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) 
+        input("please check memory usage for taosd. After checking, press enter")
+
+        #alter cache block to 8, then check alter
+        tdSql.execute('alter database db blocks 8')
+        tdSql.query('show databases')
+        tdSql.checkData(0,9,8)
+
+        #run taosdemo to occupy all cache, need to manually check memory consumption
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block2.json" % binPath) 
+        input("please check memory usage for taosd. After checking, press enter")
+
+        ##expected result the peak memory consumption should increase by around 80MB = 5 blocks of cache
+
+        ##test results
+        #2021/06/02 before:2621700K     after: 2703640K memory usage increased by 80MB = 5 block
+        #           confirm with the change in block.       Baosheng Chang
+
+    def stop(self):
+        tdSql.close()
+        tdLog.debug("%s alter block manual check finish" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/manualTest/manual_alter_comp.py

@@ -0,0 +1,126 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+
+##TODO: auto test version is currently unsupported, need to come up with 
+#       an auto test version in the future
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def getRootPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+        print(selfPath)
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+            print(projPath)
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+            print("test" + projPath)
+
+        for root, dirs, files in os.walk(projPath):
+            if ('data' in dirs and 'sim' in root):
+                rootPath = root
+
+        return rootPath
+        
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+        
+        
+
+    def run(self):
+        dnodePath = self.getRootPath()
+        os.system(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*')
+        tdSql.prepare()
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"  
+
+        #comp is at 14
+        #check disk usage when comp=2
+        tdSql.query('show databases')
+        tdSql.execute('alter database db blocks 3') # minimize the data in cache
+        tdSql.checkData(0,14,2)
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath)
+        print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data/vnode")
+        print('comp = 2')
+        input("please check disk usage for taosd. After checking, press enter")
+        
+        #removing all data file
+        os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*')
+        #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran
+        input("please check if the pervious data is being deleted. Then, press enter")
+
+        #check disk usage when comp=0
+        tdSql.prepare()
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,2)
+        tdSql.execute('alter database db comp 0')
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,0)
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath)
+        print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data")
+        print('comp = 0')
+        input("please check disk usage for taosd. After checking, press enter")
+
+        #removing all data file
+        os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*')
+        #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran
+        input("please check if the pervious data is being deleted. Then, press enter")
+
+        #check disk usage when comp=1
+        tdSql.prepare()
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,2)
+        tdSql.execute('alter database db comp 1')
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,1)
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath)
+        print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data")
+        print('comp = 1')
+        input("please check disk usage for taosd. After checking, press enter")
+
+        ##test result
+        #   2021/06/02 comp=2:13M   comp=1:57M comp=0:399M. Test past
+        #       each row entered is identical       Tester - Baosheng Chang
+
+    def stop(self):
+        tdSql.close()
+        tdLog.debug("%s alter block manual check finish" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json

@@ -0,0 +1,60 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,                   
+    "thread_count_create_tbl": 4,        
+    "result_file": "./insert_res.txt",   
+    "confirm_parameter_prompt": "no",    
+    "insert_interval": 0,         
+    "interlace_rows": 100,        
+    "num_of_records_per_req": 32766,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",                
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 3,
+            "precision": "ms",
+            "keep": 3650,
+            "minRows": 1000,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",   
+            "childtable_count": 500,   
+            "childtable_prefix": "stb_", 
+            "auto_create_table": "no",   
+            "batch_create_tbl_num": 20,   
+            "data_source": "sample",       
+            "insert_mode": "taosc",      
+            "insert_rows": 10000,       
+            "childtable_limit": 10,      
+            "childtable_offset":100,     
+            "interlace_rows": 0,         
+            "insert_interval":0,         
+            "max_sql_len": 1024000,      
+            "disorder_ratio": 0,         
+            "disorder_range": 1000,      
+            "timestamp_step": 1,         
+            "start_timestamp": "2019-10-01 00:00:00.000",  
+            "sample_format": "csv",       
+            "sample_file": "./tools/taosdemoAllTest/sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT", "count":3}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}, {"type": "BOOL"}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}

tests/pytest/tools/taosdemoAllTest/manual_block2.json

@@ -0,0 +1,60 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,                   
+    "thread_count_create_tbl": 4,        
+    "result_file": "./insert_res.txt",   
+    "confirm_parameter_prompt": "no",    
+    "insert_interval": 0,         
+    "interlace_rows": 100,        
+    "num_of_records_per_req": 32766,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",                
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 3650,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"yes",   
+            "childtable_count": 500,   
+            "childtable_prefix": "stb_", 
+            "auto_create_table": "no",   
+            "batch_create_tbl_num": 20,   
+            "data_source": "rand",       
+            "insert_mode": "taosc",      
+            "insert_rows": 100000,       
+            "childtable_limit": 500,      
+            "childtable_offset":0,     
+            "interlace_rows": 0,         
+            "insert_interval":0,         
+            "max_sql_len": 1024000,      
+            "disorder_ratio": 0,         
+            "disorder_range": 1000,      
+            "timestamp_step": 1,         
+            "start_timestamp": "now",  
+            "sample_format": "csv",       
+            "sample_file": "./sample.csv",   
+            "tags_file": "",              
+            "columns": [{"type": "INT"}],
+            "tags": [{"type": "TINYINT", "count":2}]
+        }]
+    }]
+}