docs: query forbid super table and special-table together (#30284)

* docs: query forbid super table and special-table together

* fix: taosBenchmark official case add to CI

* fix: tmq check consumer result

* docs: adjust json format and add note tips for both query

* docs: json format indent for query.json

* fix: forbid both query from v3.3.5.6
This commit is contained in:
Alex Duan 2025-03-20 15:59:07 +08:00 committed by GitHub
parent 0baea6ad8e
commit d27848fe3b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 314 additions and 22 deletions

View File

@ -355,6 +355,7 @@ Specify the configuration parameters for tag and data columns in `super_tables`
`query_times` specifies the number of times to run the query, numeric type.
**Note: from version 3.3.5.6 and beyond, simultaneous configuration for `specified_table_query` and `super_table_query` in a JSON file is no longer supported **
For other common parameters, see [General Configuration Parameters](#general-configuration-parameters)
@ -484,6 +485,15 @@ Note: Data types in the taosBenchmark configuration file must be in lowercase to
</details>
<details>
<summary>queryStb.json</summary>
```json
{{#include /TDengine/tools/taos-tools/example/queryStb.json}}
```
</details>
#### Subscription Example
<details>

View File

@ -263,6 +263,8 @@ taosBenchmark -f <json file>
其它通用参数详见 [通用配置参数](#通用配置参数)。
**说明:从 v3.3.5.6 及以上版本不再支持 json 文件中同时配置 `specified_table_query``super_table_query`**
#### 执行指定查询语句
查询指定表(可以指定超级表、子表或普通表)的配置参数在 `specified_table_query` 中设置。
@ -389,6 +391,15 @@ taosBenchmark -f <json file>
</details>
<details>
<summary>queryStb.json</summary>
```json
{{#include /TDengine/tools/taos-tools/example/queryStb.json}}
```
</details>
### 订阅 JSON 示例
<details>

View File

@ -10,7 +10,7 @@
"query_times": 100,
"query_mode": "taosc",
"specified_table_query": {
"concurrent": 3,
"threads": 3,
"sqls": [
{
"sql": "select last_row(*) from meters"

View File

@ -10,7 +10,7 @@
"query_times": 100,
"query_mode": "taosc",
"specified_table_query": {
"concurrent": 4,
"threads": 4,
"mixed_query": "yes",
"sqls": [
{

View File

@ -10,7 +10,7 @@
"query_times": 5,
"query_mode": "taosc",
"specified_table_query": {
"concurrent": 5,
"threads": 5,
"query_interval": 1000,
"mixed_query": "yes",
"batch_query": "yes",

View File

@ -10,7 +10,7 @@
"query_times": 5,
"query_mode": "taosc",
"specified_table_query": {
"concurrent": 5,
"threads": 5,
"query_interval": 100,
"mixed_query": "yes",
"batch_query": "no",

View File

@ -10,7 +10,7 @@
"query_times": 100,
"query_mode": "rest",
"specified_table_query": {
"concurrent": 3,
"threads": 3,
"mixed_query": "yes",
"sqls": [
{

View File

@ -10,7 +10,7 @@
"query_times": 100,
"query_mode": "rest",
"specified_table_query": {
"concurrent": 3,
"threads": 3,
"sqls": [
{
"sql": "select last_row(*) from meters"

View File

@ -12,7 +12,7 @@
"query_mode": "taosc",
"super_table_query": {
"stblname": "meters",
"concurrent": 3,
"threads": 3,
"query_interval": 0,
"sqls": [
{

View File

@ -12,7 +12,7 @@
"query_mode": "rest",
"super_table_query": {
"stblname": "meters",
"concurrent": 3,
"threads": 3,
"query_interval": 0,
"sqls": [
{

View File

@ -0,0 +1,257 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import json
import sys
import os
import time
import datetime
import platform
import subprocess
import frame
import frame.eos
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
# reomve single and double quotation
def removeQuotation(origin):
value = ""
for c in origin:
if c != '\'' and c != '"':
value += c
return value
class TDTestCase(TBase):
def caseDescription(self):
"""
taosBenchmark query->Basic test cases
"""
def runSeconds(self, command, timeout = 180):
tdLog.info(f"runSeconds {command} ...")
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.wait(timeout)
# get output
output = process.stdout.read().decode(encoding="gbk")
error = process.stderr.read().decode(encoding="gbk")
return output, error
def getKeyValue(self, content, key, end):
# find key
s = content.find(key)
if s == -1:
return False,""
# skip self
s += len(key)
# skip blank
while s < len(content):
if content[s] != " ":
break
s += 1
# end check
if s + 1 == len(content):
return False, ""
# find end
if len(end) == 0:
e = -1
else:
e = content.find(end, s)
# get value
if e == -1:
value = content[s : ]
else:
value = content[s : e]
return True, value
def getDbRows(self, times):
sql = f"select count(*) from test.meters"
tdSql.waitedQuery(sql, 1, times)
dbRows = tdSql.getData(0, 0)
return dbRows
def checkItem(self, output, key, end, expect, equal):
ret, value = self.getKeyValue(output, key, end)
if ret == False:
tdLog.exit(f"not found key:{key}. end:{end} output:\n{output}")
fval = float(value)
# compare
if equal and fval != expect:
tdLog.exit(f"check not expect. expect:{expect} real:{fval}, key:'{key}' end:'{end}' output:\n{output}")
elif equal == False and fval <= expect:
tdLog.exit(f"failed because {fval} <= {expect}, key:'{key}' end:'{end}' output:\n{output}")
else:
# succ
if equal:
tdLog.info(f"check successfully. key:'{key}' expect:{expect} real:{fval}")
else:
tdLog.info(f"check successfully. key:'{key}' {fval} > {expect}")
def checkAfterRun(self, benchmark, jsonFile, specMode, tbCnt):
# run
cmd = f"{benchmark} -f {jsonFile}"
output, error = self.runSeconds(cmd)
if specMode :
label = "specified_table_query"
else:
label = "super_table_query"
#
# check insert result
#
with open(jsonFile, "r") as file:
data = json.load(file)
queryTimes = data["query_times"]
# contineIfFail
try:
continueIfFail = data["continue_if_fail"]
except:
continueIfFail = "no"
threads = data[label]["threads"]
sqls = data[label]["sqls"]
# batch_query
try:
batchQuery = data[label]["batch_query"]
except:
batchQuery = "no"
# mixed_query
try:
mixedQuery = data[label]["mixed_query"]
except:
mixedQuery = "no"
tdLog.info(f"queryTimes={queryTimes} threads={threads} mixedQuery={mixedQuery} "
f"batchQuery={batchQuery} len(sqls)={len(sqls)} label={label}\n")
totalQueries = 0
threadQueries = 0
QPS = 10
if continueIfFail.lower() == "yes":
allEnd = " "
else:
allEnd = "\n"
if specMode and mixedQuery.lower() != "yes":
# spec
threadQueries = queryTimes * threads
totalQueries = queryTimes * threads * len(sqls)
threadKey = f"complete query with {threads} threads and "
qpsKey = "QPS: "
avgKey = "query delay avg: "
minKey = "min:"
else:
# spec mixed or super
if specMode:
totalQueries = queryTimes * len(sqls)
# spec mixed
if batchQuery.lower() == "yes":
# batch
threadQueries = len(sqls)
QPS = 2
else:
threadQueries = totalQueries
else:
# super
totalQueries = queryTimes * len(sqls) * tbCnt
threadQueries = totalQueries
nSql = len(sqls)
if specMode and nSql < threads :
tdLog.info(f"set threads = {nSql} because len(sqls) < threads")
threads = nSql
threadKey = f"using {threads} threads complete query "
qpsKey = ""
avgKey = "avg delay:"
minKey = "min delay:"
items = [
[threadKey, " ", threadQueries, True],
[qpsKey, " ", 5, False], # qps need > 1
[avgKey, "s", 0, False],
[minKey, "s", 0, False],
["max: ", "s", 0, False],
["p90: ", "s", 0, False],
["p95: ", "s", 0, False],
["p99: ", "s", 0, False],
["INFO: Spend ", " ", 0, False],
["completed total queries: ", ",", totalQueries, True],
["the QPS of all threads:", allEnd, QPS , False] # all qps need > 5
]
# check
for item in items:
if len(item[0]) > 0:
self.checkItem(output, item[0], item[1], item[2], item[3])
# tmq check
def checkTmqJson(self, benchmark, json):
OK_RESULT = "Consumed total msgs: 30, total rows: 300000"
cmd = benchmark + " -f " + json
output,error = frame.eos.run(cmd, 600)
if output.find(OK_RESULT) != -1:
tdLog.info(f"succ: {cmd} found '{OK_RESULT}'")
else:
tdLog.exit(f"failed: {cmd} not found {OK_RESULT} in:\n{output} \nerror:{error}")
def run(self):
tbCnt = 10
benchmark = etool.benchMarkFile()
# insert
json = "../../tools/taos-tools/example/insert.json"
self.insertBenchJson(json, checkStep=True)
# query
json = "../../tools/taos-tools/example/query.json"
self.checkAfterRun(benchmark, json, True, tbCnt)
json = "../../tools/taos-tools/example/queryStb.json"
self.checkAfterRun(benchmark, json, False, tbCnt)
# tmq
json = "../../tools/taos-tools/example/tmq.json"
self.checkTmqJson(benchmark, json)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

View File

@ -77,6 +77,7 @@
#
# benchmark 64 cases
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/websiteCase.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/rest_insert_alltypes_json.py -R
,,n,army,python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson-mixed-query.py -R
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json.py

View File

@ -27,7 +27,7 @@
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 1000,
"childtable_count": 10,
"childtable_prefix": "d",
"auto_create_table": "no",
"batch_create_tbl_num": 5,

View File

@ -6,13 +6,13 @@
"user": "root",
"password": "taosdata",
"confirm_parameter_prompt": "no",
"continue_if_fail": "yes",
"continue_if_fail": "yes",
"databases": "test",
"query_times": 10,
"query_mode": "taosc",
"specified_table_query": {
"query_interval": 1,
"concurrent": 3,
"threads": 3,
"sqls": [
{
"sql": "select last_row(*) from meters",
@ -23,16 +23,5 @@
"result": "./query_res1.txt"
}
]
},
"super_table_query": {
"stblname": "meters",
"query_interval": 1,
"threads": 3,
"sqls": [
{
"sql": "select last_row(ts) from xxxx",
"result": "./query_res2.txt"
}
]
}
}

View File

@ -0,0 +1,24 @@
{
"filetype": "query",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"confirm_parameter_prompt": "no",
"continue_if_fail": "yes",
"databases": "test",
"query_times": 10,
"query_mode": "taosc",
"super_table_query": {
"stblname": "meters",
"query_interval": 1,
"threads": 3,
"sqls": [
{
"sql": "select last_row(ts) from xxxx",
"result": "./query_res2.txt"
}
]
}
}