xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[ci skip] update scirpt for csv generator

This commit is contained in:
Ping Xiao 2021-08-19 14:13:17 +08:00
parent da8ebbef3e
commit eb4e906737
1 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
tests/pytest/insert/insertFromCSVPerformance.py
View File

@ -28,7 +28,7 @@ class insertFromCSVPerformace:
self.tbName = tbName self.tbName = tbName
self.branchName = branchName self.branchName = branchName
self.type = buildType self.type = buildType
self.ts = 1500074556514 self.ts = 1500000000000
self.host = "127.0.0.1" self.host = "127.0.0.1"
self.user = "root" self.user = "root"
self.password = "taosdata" self.password = "taosdata"
@ -46,13 +46,20 @@ class insertFromCSVPerformace:
config = self.config) config = self.config)
def writeCSV(self): def writeCSV(self):
with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: tsset = set()
rows = 0
with open('test4.csv','w', encoding='utf-8', newline='') as csvFile:
writer = csv.writer(csvFile, dialect='excel') writer = csv.writer(csvFile, dialect='excel')
for i in range(1000000): while True:
newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) newTimestamp = self.ts + random.randint(1, 10) * 10000000000 + random.randint(1, 10) * 1000000000 + random.randint(1, 10) * 100000000 + random.randint(1, 10) * 10000000 + random.randint(1, 10) * 1000000 + random.randint(1, 10) * 100000 + random.randint(1, 10) * 10000 + random.randint(1, 10) * 1000 + random.randint(1, 10) * 100 + random.randint(1, 10) * 10 + random.randint(1, 10)
d = datetime.datetime.fromtimestamp(newTimestamp / 1000) if newTimestamp not in tsset:
dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) tsset.add(newTimestamp)
writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
rows += 1
if rows == 2000000:
break
def removCSVHeader(self): def removCSVHeader(self):
data = pd.read_csv("ordered.csv") data = pd.read_csv("ordered.csv")
@ -71,7 +78,9 @@ class insertFromCSVPerformace:
cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
startTime = time.time() startTime = time.time()
cursor.execute("insert into t1 file 'outoforder.csv'") cursor.execute("insert into t1 file 'outoforder.csv'")
totalTime += time.time() - startTime totalTime += time.time() - startTime
time.sleep(1)
out_of_order_time = (float) (totalTime / 10) out_of_order_time = (float) (totalTime / 10)
print("Out of Order - Insert time: %f" % out_of_order_time) print("Out of Order - Insert time: %f" % out_of_order_time)
@ -81,7 +90,8 @@ class insertFromCSVPerformace:
cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
startTime = time.time() startTime = time.time()
cursor.execute("insert into t2 file 'ordered.csv'") cursor.execute("insert into t2 file 'ordered.csv'")
totalTime += time.time() - startTime totalTime += time.time() - startTime
time.sleep(1)
in_order_time = (float) (totalTime / 10) in_order_time = (float) (totalTime / 10)
print("In order - Insert time: %f" % in_order_time) print("In order - Insert time: %f" % in_order_time)