[TD-3602]<test>: update perf test script

tests/perftest-scripts/perftest-query.sh

@@ -71,21 +71,8 @@ function runQueryPerfTest {
 	python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
 
 	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
-
-	taosdemo -f /home/ubuntu/pxiao/insert.json > taosdemoperf.txt
-
-	CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'`
-	INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'`
-	REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'`
-	delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'`
-	AVGDELAY=`echo ${delay:0:${#delay}-3}`
-	delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'`		
-	MAXDELAY=`echo ${delay:0:${#delay}-3}`	
-	delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'`
-	MINDELAY=`echo ${delay:0:${#delay}-2}`	
 	
-	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY  | tee -a $PERFORMANCE_TEST_REPORT
+	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
-	[ -f taosdemoperf.txt ] && rm taosdemoperf.txt
 }
 
 

tests/pytest/tools/taosdemoPerformance.py

@@ -11,26 +11,16 @@
 
 # -*- coding: utf-8 -*-
 
-import sys
 import taos
-import time
-import datetime
-import csv
-import random
 import pandas as pd
 import argparse
 import os.path
+import json
 
 class taosdemoPerformace:
-    def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay):
+    def __init__(self, commitID, dbName):
         self.commitID = commitID
-        self.dbName = dbName
+        self.dbName = dbName        
-        self.createTableTime = createTableTime
-        self.insertRecordsTime = insertRecordsTime
-        self.recordsPerSecond = recordsPerSecond
-        self.avgDelay = avgDelay
-        self.maxDelay = maxDelay
-        self.minDelay = minDelay
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -69,11 +59,14 @@ class taosdemoPerformace:
             "start_timestamp": "2020-10-01 00:00:00.000",
             "sample_format": "csv",
             "sample_file": "./sample.csv",
-            "tags_file": "",
+            "tags_file": "",            
-            "columns": [{
+            "columns": [
-                "columns": [{"type": "INT", "count": 4}],
+                {"type": "INT", "count": 4}
-            }],
+            ],            
-            "tags": [{"type": "INT", "count":1}, {"type": "BINARY", "len": 16}]
+            "tags": [
+                {"type": "INT", "count":1}, 
+                {"type": "BINARY", "len": 16}
+            ]
         }
 
         stables = []
@@ -97,10 +90,35 @@ class taosdemoPerformace:
             "confirm_parameter_prompt": "no",
             "insert_interval": 0,
             "num_of_records_per_req": 30000,
-            "databases": [db]
+            "databases": [db]        
         }
 
-        return insert_data
+        insert_json_file = f"/tmp/insert.json"
+
+        with open(insert_json_file, 'w') as f:
+            json.dump(insert_data, f)
+        return insert_json_file
+
+    def getCMDOutput(self, cmd):
+        cmd = os.popen(cmd)
+        output = cmd.read()
+        cmd.close()
+        return output
+
+    def insertData(self):        
+        os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson())
+        self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'")
+        self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'")
+        self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'")
+        self.commitID = self.getCMDOutput("git rev-parse --short HEAD")
+        delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'")
+        self.avgDelay = delay[:-4]
+        delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'")
+        self.maxDelay = delay[:-4]
+        delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") 
+        self.minDelay = delay[:-3]
+
+        os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt")
 
     def createTablesAndStoreData(self):
         cursor = self.conn.cursor()
@@ -109,13 +127,14 @@ class taosdemoPerformace:
         cursor.execute("use %s" % self.dbName)
         cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)")
         print("==================== taosdemo performance ====================")
-        print("create tables time: %f" % self.createTableTime)
+        print("create tables time: %f" % float(self.createTableTime))
-        print("insert records time: %f" % self.insertRecordsTime)
+        print("insert records time: %f" % float(self.insertRecordsTime))
-        print("records per second: %f" % self.recordsPerSecond)
+        print("records per second: %f" % float(self.recordsPerSecond))
-        print("avg delay: %f" % self.avgDelay)
+        print("avg delay: %f" % float(self.avgDelay))
-        print("max delay: %f" % self.maxDelay)
+        print("max delay: %f" % float(self.maxDelay))
-        print("min delay: %f" % self.minDelay)        
+        print("min delay: %f" % float(self.minDelay)) 
-        cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay))
+        cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % 
+        (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay)))
         cursor.execute("drop database if exists %s" % self.insertDB)
 
         cursor.close()
@@ -125,7 +144,7 @@ if __name__ == '__main__':
     parser.add_argument(
         '-c',
         '--commit-id',
-        action='store',        
+        action='store',
         type=str,
         help='git commit id (default: null)')
     parser.add_argument(
@@ -135,46 +154,9 @@ if __name__ == '__main__':
         default='perf',
         type=str,
         help='Database name to be created (default: perf)')
-    parser.add_argument(
-        '-t',
-        '--create-table',
-        action='store',        
-        type=float,
-        help='create table time')
-    parser.add_argument(
-        '-i',
-        '--insert-records',
-        action='store',        
-        type=float,
-        help='insert records time')
-    parser.add_argument(
-        '-r',
-        '---records-per-second',
-        action='store',        
-        type=float,
-        help='records per request')
-    parser.add_argument(
-        '-avg',
-        '---avg-delay',
-        action='store',        
-        type=float,
-        help='avg delay')
-    parser.add_argument(
-        '-max',
-        '---max-delay',
-        action='store',        
-        type=float,
-        help='max delay')
-    parser.add_argument(
-        '-min',
-        '---min-delay',
-        action='store',        
-        type=float,
-        help='min delay')
 
-    
     args = parser.parse_args()
 
-    perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, 
+    perftest = taosdemoPerformace(args.commit_id, args.database_name) 
-                        args.avg_delay, args.max_delay, args.min_delay) 
+    perftest.insertData()
     perftest.createTablesAndStoreData()