update
This commit is contained in:
wenzhouwww@live.cn
parent
ca00459abb
commit
59f50ae8c5
|
|
@ -37,6 +37,7 @@ import requests
|
|||
# from guppy import hpy
|
||||
import gc
|
||||
import taos
|
||||
from taos.tmq import *
|
||||
|
||||
|
||||
from .shared.types import TdColumns, TdTags
|
||||
|
|
@ -675,9 +676,10 @@ class AnyState:
|
|||
CAN_DROP_DB = 2
|
||||
CAN_CREATE_FIXED_SUPER_TABLE = 3
|
||||
CAN_CREATE_STREAM = 3 # super table must exists
|
||||
CAN_CREATE_TOPIC = 3 # super table must exists
|
||||
CAN_CREATE_CONSUMERS = 3
|
||||
CAN_DROP_FIXED_SUPER_TABLE = 4
|
||||
CAN_ADD_DATA = 5
|
||||
CAN_DROP_STREAM = 5
|
||||
CAN_READ_DATA = 6
|
||||
CAN_DELETE_DATA = 6
|
||||
|
||||
|
|
@ -730,6 +732,12 @@ class AnyState:
|
|||
return False
|
||||
return self._info[self.CAN_DROP_FIXED_SUPER_TABLE]
|
||||
|
||||
def canCreateTopic(self):
|
||||
return self._info[self.CAN_CREATE_TOPIC]
|
||||
|
||||
def canCreateConsumers(self):
|
||||
return self._info[self.CAN_CREATE_CONSUMERS]
|
||||
|
||||
def canCreateStream(self):
|
||||
return self._info[self.CAN_CREATE_STREAM]
|
||||
|
||||
|
|
@ -1679,9 +1687,22 @@ class TaskDropDb(StateTransitionTask):
|
|||
return state.canDropDb()
|
||||
|
||||
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
|
||||
|
||||
# drop topics before drop db
|
||||
|
||||
if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()):
|
||||
|
||||
self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None)
|
||||
self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName)
|
||||
|
||||
self.execWtSql(wt, "drop database {}".format(self._db.getName()))
|
||||
|
||||
Logging.debug("[OPS] database dropped at {}".format(time.time()))
|
||||
|
||||
|
||||
'''
|
||||
# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ）
|
||||
|
||||
class TaskCreateStream(StateTransitionTask):
|
||||
|
||||
@classmethod
|
||||
|
|
@ -1707,9 +1728,9 @@ class TaskCreateStream(StateTransitionTask):
|
|||
# wt.execSql("use db") # should always be in place
|
||||
|
||||
# create stream
|
||||
'''
|
||||
CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
|
||||
'''
|
||||
|
||||
# CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
|
||||
|
||||
stbname =sTable.getName()
|
||||
sub_tables = sTable.getRegTables(wt.getDbConn())
|
||||
aggExpr = Dice.choice([
|
||||
|
|
@ -1735,7 +1756,7 @@ class TaskCreateStream(StateTransitionTask):
|
|||
if sub_tables: # if not empty
|
||||
sub_tbname = sub_tables[0]
|
||||
# create stream with query above sub_table
|
||||
stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} where ts <now and ts >now -1h PARTITION BY tbname INTERVAL(5s) SLIDING(3s) FILL (prev) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname)
|
||||
stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname)
|
||||
try:
|
||||
self.execWtSql(wt, stream_sql)
|
||||
Logging.debug("[OPS] stream is creating at {}".format(time.time()))
|
||||
|
|
@ -1749,7 +1770,7 @@ class TaskCreateStream(StateTransitionTask):
|
|||
pass
|
||||
|
||||
else:
|
||||
stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} where ts <now and ts >now -1h PARTITION BY tbname INTERVAL(5s) SLIDING(3s) FILL (prev) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname)
|
||||
stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname)
|
||||
|
||||
try:
|
||||
self.execWtSql(wt, stream_sql)
|
||||
|
|
@ -1759,8 +1780,142 @@ class TaskCreateStream(StateTransitionTask):
|
|||
if errno in [0x03f0]: # stream already exists
|
||||
# stream need drop before drop table
|
||||
pass
|
||||
'''
|
||||
|
||||
|
||||
class TaskCreateTopic(StateTransitionTask):
|
||||
|
||||
@classmethod
|
||||
def getEndState(cls):
|
||||
return StateHasData()
|
||||
|
||||
@classmethod
|
||||
def canBeginFrom(cls, state: AnyState):
|
||||
return state.canCreateTopic()
|
||||
|
||||
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
|
||||
dbname = self._db.getName()
|
||||
|
||||
sub_topic_name = dbname+ '_sub_topic'
|
||||
super_topic_name = dbname+ '_super_topic'
|
||||
stable_topic = dbname+ '_stable_topic'
|
||||
db_topic = 'database_' + dbname+ '_topics'
|
||||
if not self._db.exists(wt.getDbConn()):
|
||||
Logging.debug("Skipping task, no DB yet")
|
||||
return
|
||||
|
||||
sTable = self._db.getFixedSuperTable() # type: TdSuperTable
|
||||
# wt.execSql("use db") # should always be in place
|
||||
|
||||
# create topic
|
||||
|
||||
# create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1;
|
||||
|
||||
stbname =sTable.getName()
|
||||
sub_tables = sTable.getRegTables(wt.getDbConn())
|
||||
scalarExpr = Dice.choice([ '*','speed','color',
|
||||
'abs(speed)',
|
||||
'acos(speed)',
|
||||
'asin(speed)',
|
||||
'atan(speed)',
|
||||
'ceil(speed)',
|
||||
'cos(speed)',
|
||||
'cos(speed)',
|
||||
'floor(speed)',
|
||||
'log(speed,2)',
|
||||
'pow(speed,2)',
|
||||
'round(speed)',
|
||||
'sin(speed)',
|
||||
'sqrt(speed)',
|
||||
'char_length(color)',
|
||||
'concat(color,color)',
|
||||
'concat_ws(" ", color,color," ")',
|
||||
'length(color)',
|
||||
'lower(color)',
|
||||
'ltrim(color)',
|
||||
'substr(color , 2)',
|
||||
'upper(color)',
|
||||
'cast(speed as double)',
|
||||
'cast(ts as bigint)',
|
||||
|
||||
]) # TODO: add more from 'top'
|
||||
if Dice.throw(3)==0:
|
||||
|
||||
if sub_tables:
|
||||
|
||||
if sub_tables: # if not empty
|
||||
sub_tbname = sub_tables[0]
|
||||
# create stream with query above sub_table
|
||||
topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname)
|
||||
try:
|
||||
self.execWtSql(wt, "use {}".format(dbname))
|
||||
self.execWtSql(wt, topic_sql)
|
||||
Logging.debug("[OPS] topic is creating at {}".format(time.time()))
|
||||
except taos.error.ProgrammingError as err:
|
||||
errno = Helper.convertErrno(err.errno)
|
||||
if errno in [0x03f0]: # topic already exists
|
||||
# topic need drop before drop table
|
||||
pass
|
||||
|
||||
else:
|
||||
pass
|
||||
|
||||
else:
|
||||
topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname)
|
||||
try:
|
||||
self.execWtSql(wt, "use {}".format(dbname))
|
||||
self.execWtSql(wt, topic_sql)
|
||||
Logging.debug("[OPS] subquery topic is creating at {}".format(time.time()))
|
||||
except taos.error.ProgrammingError as err:
|
||||
errno = Helper.convertErrno(err.errno)
|
||||
if errno in [0x03f0]: # topic already exists
|
||||
# topic need drop before drop table
|
||||
pass
|
||||
elif Dice.throw(3)==1:
|
||||
topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname)
|
||||
try:
|
||||
self.execWtSql(wt, "use {}".format(dbname))
|
||||
self.execWtSql(wt, topic_sql)
|
||||
Logging.debug("[OPS] stable topic is creating at {}".format(time.time()))
|
||||
except taos.error.ProgrammingError as err:
|
||||
errno = Helper.convertErrno(err.errno)
|
||||
if errno in [0x03f0]: # topic already exists
|
||||
# topic need drop before drop table
|
||||
pass
|
||||
elif Dice.throw(3)==2:
|
||||
topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname)
|
||||
try:
|
||||
self.execWtSql(wt, "use {}".format(dbname))
|
||||
self.execWtSql(wt, topic_sql)
|
||||
Logging.debug("[OPS] db topic is creating at {}".format(time.time()))
|
||||
except taos.error.ProgrammingError as err:
|
||||
errno = Helper.convertErrno(err.errno)
|
||||
if errno in [0x03f0]: # topic already exists
|
||||
# topic need drop before drop table
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
class TaskCreateConsumers(StateTransitionTask):
|
||||
|
||||
@classmethod
|
||||
def getEndState(cls):
|
||||
return StateHasData()
|
||||
|
||||
@classmethod
|
||||
def canBeginFrom(cls, state: AnyState):
|
||||
return state.canCreateConsumers()
|
||||
|
||||
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
|
||||
dbname = self._db.getName()
|
||||
|
||||
sTable = self._db.getFixedSuperTable() # type: TdSuperTable
|
||||
# wt.execSql("use db") # should always be in place
|
||||
|
||||
# create Consumers
|
||||
if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task
|
||||
if sTable.hasTopics(wt.getDbConn()):
|
||||
sTable.createConsumer(wt.getDbConn(),random.randint(1,10))
|
||||
|
||||
|
||||
class TaskCreateSuperTable(StateTransitionTask):
|
||||
|
|
@ -1780,9 +1935,6 @@ class TaskCreateSuperTable(StateTransitionTask):
|
|||
sTable = self._db.getFixedSuperTable() # type: TdSuperTable
|
||||
# wt.execSql("use db") # should always be in place
|
||||
|
||||
if sTable.hasStreams(wt.getDbConn()) or sTable.hasStreamTables(wt.getDbConn()):
|
||||
sTable.dropStreams(wt.getDbConn())
|
||||
sTable.dropStreamTables(wt.getDbConn())
|
||||
sTable.create(wt.getDbConn(),
|
||||
{'ts': TdDataType.TIMESTAMP, 'speed': TdDataType.INT, 'color': TdDataType.BINARY16}, {
|
||||
'b': TdDataType.BINARY200, 'f': TdDataType.FLOAT},
|
||||
|
|
@ -1797,6 +1949,8 @@ class TdSuperTable:
|
|||
def __init__(self, stName, dbName):
|
||||
self._stName = stName
|
||||
self._dbName = dbName
|
||||
self._consumerLists = {}
|
||||
self._ConsumerInsts = []
|
||||
|
||||
def getName(self):
|
||||
return self._stName
|
||||
|
|
@ -1806,6 +1960,12 @@ class TdSuperTable:
|
|||
dbName = self._dbName
|
||||
if self.exists(dbc) : # if myself exists
|
||||
fullTableName = dbName + '.' + self._stName
|
||||
if self.hasStreams(dbc):
|
||||
self.dropStreams(dbc)
|
||||
self.dropStreamTables(dbc)
|
||||
if self.hasTopics(dbc):
|
||||
self.dropTopics(dbName,None)
|
||||
self.dropTopics(dbName,self._stName)
|
||||
try:
|
||||
dbc.execute("DROP TABLE {}".format(fullTableName))
|
||||
except taos.error.ProgrammingError as err:
|
||||
|
|
@ -1843,12 +2003,19 @@ class TdSuperTable:
|
|||
|
||||
if dbc.existsSuperTable(self._stName):
|
||||
if dropIfExists:
|
||||
if self.hasStreams(dbc):
|
||||
self.dropStreams(dbc)
|
||||
self.dropStreamTables(dbc)
|
||||
|
||||
# drop topics before drop stables
|
||||
if self.hasTopics(dbc):
|
||||
self.dropTopics(dbc,self._dbName,None)
|
||||
self.dropTopics(dbc,self._dbName,self._stName )
|
||||
|
||||
|
||||
dbc.execute("DROP TABLE {}".format(fullTableName))
|
||||
|
||||
pass
|
||||
|
||||
|
||||
pass
|
||||
# dbc.execute("DROP TABLE {}".format(fullTableName))
|
||||
else: # error
|
||||
raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName))
|
||||
|
|
@ -1863,7 +2030,47 @@ class TdSuperTable:
|
|||
)
|
||||
else:
|
||||
sql += " TAGS (dummy int) "
|
||||
dbc.execute(sql)
|
||||
dbc.execute(sql)
|
||||
|
||||
def createConsumer(self, dbc: DbConn , Consumer_nums):
|
||||
|
||||
def generateConsumer(current_topic_list):
|
||||
conf = TaosTmqConf()
|
||||
conf.set("group.id", "tg2")
|
||||
conf.set("td.connect.user", "root")
|
||||
conf.set("td.connect.pass", "taosdata")
|
||||
conf.set("enable.auto.commit", "true")
|
||||
def tmq_commit_cb_print(tmq, resp, offset, param=None):
|
||||
print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}")
|
||||
conf.set_auto_commit_cb(tmq_commit_cb_print, None)
|
||||
consumer = conf.new_consumer()
|
||||
topic_list = TaosTmqList()
|
||||
for topic in current_topic_list:
|
||||
topic_list.append(topic)
|
||||
consumer.subscribe(topic_list)
|
||||
time.sleep(5) # consumer work only 5 sec ,and then it will exit
|
||||
try:
|
||||
consumer.unsubscribe()
|
||||
except Exception as e :
|
||||
pass
|
||||
return
|
||||
|
||||
# mulit Consumer
|
||||
current_topic_list = self.getTopicLists(dbc)
|
||||
for i in range(Consumer_nums):
|
||||
consumer_inst = threading.Thread(target=generateConsumer, args=(current_topic_list,))
|
||||
self._ConsumerInsts.append(consumer_inst)
|
||||
|
||||
for ConsumerInst in self._ConsumerInsts:
|
||||
ConsumerInst.start()
|
||||
for ConsumerInst in self._ConsumerInsts:
|
||||
ConsumerInst.join()
|
||||
|
||||
def getTopicLists(self, dbc: DbConn):
|
||||
dbc.query("show topics ")
|
||||
topics = dbc.getQueryResult()
|
||||
topicLists = [v[0] for v in topics]
|
||||
return topicLists
|
||||
|
||||
def getRegTables(self, dbc: DbConn):
|
||||
dbName = self._dbName
|
||||
|
|
@ -1878,12 +2085,7 @@ class TdSuperTable:
|
|||
return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation
|
||||
|
||||
def hasRegTables(self, dbc: DbConn):
|
||||
# print(self._stName)
|
||||
# dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName))
|
||||
# print(dbc.getQueryResult())
|
||||
if self.hasStreamTables(dbc) or self.hasStreams(dbc):
|
||||
if self.dropStreams(dbc):
|
||||
self.dropStreamTables(dbc)
|
||||
|
||||
if dbc.existsSuperTable(self._stName):
|
||||
|
||||
return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0
|
||||
|
|
@ -1896,7 +2098,41 @@ class TdSuperTable:
|
|||
|
||||
def hasStreams(self,dbc: DbConn):
|
||||
return dbc.query("show streams") > 0
|
||||
|
||||
|
||||
def hasTopics(self,dbc: DbConn):
|
||||
|
||||
return dbc.query("show topics") > 0
|
||||
|
||||
def dropTopics(self,dbc: DbConn , dbname=None,stb_name=None):
|
||||
dbc.query("show topics ")
|
||||
topics = dbc.getQueryResult()
|
||||
|
||||
if dbname !=None and stb_name == None :
|
||||
|
||||
for topic in topics:
|
||||
if dbname in topic[0] and topic[0].startswith("database"):
|
||||
try:
|
||||
dbc.execute('drop topic {}'.format(topic[0]))
|
||||
except taos.error.ProgrammingError as err:
|
||||
errno = Helper.convertErrno(err.errno)
|
||||
if errno in [0x03EB]: # Topic subscribed cannot be dropped
|
||||
pass
|
||||
# for subsript in subscriptions:
|
||||
|
||||
else:
|
||||
pass
|
||||
|
||||
pass
|
||||
return True
|
||||
elif dbname !=None and stb_name!= None:
|
||||
for topic in topics:
|
||||
if topic[0].startswith(self._dbName) and topic[0].endswith('topic'):
|
||||
dbc.execute('drop topic {}'.format(topic[0]))
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
pass
|
||||
|
||||
def dropStreams(self,dbc:DbConn):
|
||||
dbc.query("show streams ")
|
||||
Streams = dbc.getQueryResult()
|
||||
|
|
@ -1912,7 +2148,7 @@ class TdSuperTable:
|
|||
StreamTables = dbc.getQueryResult()
|
||||
|
||||
for StreamTable in StreamTables:
|
||||
if self.dropStreams:
|
||||
if self.dropStreams(dbc):
|
||||
dbc.execute('drop table {}.{}'.format(self._dbName,StreamTable[0]))
|
||||
|
||||
return not dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName))
|
||||
|
|
@ -2083,11 +2319,7 @@ class TdSuperTable:
|
|||
'sample(speed,5)',
|
||||
'STATECOUNT(speed,"LT",1)',
|
||||
'STATEDURATION(speed,"LT",1)',
|
||||
'twa(speed)'
|
||||
|
||||
|
||||
|
||||
|
||||
'twa(speed)'
|
||||
|
||||
]) # TODO: add more from 'top'
|
||||
|
||||
|
|
@ -2194,6 +2426,7 @@ class TaskDropSuperTable(StateTransitionTask):
|
|||
for i in tblSeq:
|
||||
regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i)
|
||||
try:
|
||||
|
||||
self.execWtSql(wt, "drop table {}.{}".
|
||||
format(self._db.getName(), regTableName)) # nRows always 0, like MySQL
|
||||
except taos.error.ProgrammingError as err:
|
||||
|
|
@ -2209,6 +2442,17 @@ class TaskDropSuperTable(StateTransitionTask):
|
|||
|
||||
# Drop the super table itself
|
||||
tblName = self._db.getFixedSuperTableName()
|
||||
|
||||
# drop streams before drop stables
|
||||
if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()):
|
||||
self._db.getFixedSuperTable().dropStreams(wt.getDbConn())
|
||||
self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn())
|
||||
|
||||
# drop topics before drop stables
|
||||
if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()):
|
||||
self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None)
|
||||
self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName)
|
||||
|
||||
try:
|
||||
self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName))
|
||||
except taos.error.ProgrammingError as err:
|
||||
|
|
|
|||
Loading…
Reference in New Issue