131 lines
4.9 KiB
Python
131 lines
4.9 KiB
Python
###################################################################
|
|
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This file is proprietary and confidential to TAOS Technologies.
|
|
# No part of this file may be reproduced, stored, transmitted,
|
|
# disclosed or used in any form or by any means other than as
|
|
# expressly provided by the written permission from Jianhui Tao
|
|
#
|
|
###################################################################
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import sys
|
|
import taos
|
|
import time
|
|
import datetime
|
|
import csv
|
|
import random
|
|
import pandas as pd
|
|
import argparse
|
|
import os.path
|
|
|
|
class insertFromCSVPerformace:
|
|
def __init__(self, commitID, dbName, stbName, branchName):
|
|
self.commitID = commitID
|
|
self.dbName = dbName
|
|
self.stbName = stbName
|
|
self.branchName = branchName
|
|
self.ts = 1500074556514
|
|
self.host = "127.0.0.1"
|
|
self.user = "root"
|
|
self.password = "taosdata"
|
|
self.config = "/etc/perf"
|
|
self.conn = taos.connect(
|
|
self.host,
|
|
self.user,
|
|
self.password,
|
|
self.config)
|
|
|
|
def writeCSV(self):
|
|
with open('test3.csv','w', encoding='utf-8', newline='') as csvFile:
|
|
writer = csv.writer(csvFile, dialect='excel')
|
|
for i in range(1000000):
|
|
newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000)
|
|
d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
|
|
dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
|
|
writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
|
|
|
|
def removCSVHeader(self):
|
|
data = pd.read_csv("ordered.csv")
|
|
data = data.drop([0])
|
|
data.to_csv("ordered.csv", header = False, index = False)
|
|
|
|
def createTables(self):
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("create database if not exists %s" % self.dbName)
|
|
cursor.execute("use %s" % self.dbName)
|
|
cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName)
|
|
cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName))
|
|
|
|
cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
|
|
cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
|
|
|
|
cursor.close()
|
|
|
|
def run(self):
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("use %s" % self.dbName)
|
|
print("==================== CSV insert performance ====================")
|
|
|
|
totalTime = 0
|
|
for i in range(10):
|
|
cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
|
|
startTime = time.time()
|
|
cursor.execute("insert into t1 file 'outoforder.csv'")
|
|
totalTime += time.time() - startTime
|
|
cursor.execute("drop table if exists t1")
|
|
out_of_order_time = (float) (totalTime / 10)
|
|
print("Out of Order - Insert time: %f" % out_of_order_time)
|
|
|
|
totalTime = 0
|
|
for i in range(10):
|
|
cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
|
|
startTime = time.time()
|
|
cursor.execute("insert into t2 file 'ordered.csv'")
|
|
totalTime += time.time() - startTime
|
|
cursor.execute("drop table if exists t2")
|
|
|
|
in_order_time = (float) (totalTime / 10)
|
|
print("In order - Insert time: %f" % in_order_time)
|
|
cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'-c',
|
|
'--commit-id',
|
|
action='store',
|
|
default='null',
|
|
type=str,
|
|
help='git commit id (default: null)')
|
|
parser.add_argument(
|
|
'-d',
|
|
'--database-name',
|
|
action='store',
|
|
default='perf',
|
|
type=str,
|
|
help='Database name to be created (default: perf)')
|
|
parser.add_argument(
|
|
'-t',
|
|
'--stable-name',
|
|
action='store',
|
|
default='csv_insert',
|
|
type=str,
|
|
help='Database name to be created (default: csv_insert)')
|
|
parser.add_argument(
|
|
'-b',
|
|
'--branch-name',
|
|
action='store',
|
|
default='develop',
|
|
type=str,
|
|
help='branch name (default: develop)')
|
|
|
|
args = parser.parse_args()
|
|
perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name)
|
|
|
|
perftest.createTables()
|
|
perftest.run() |