From cdf8ea13893cd43deb096b6b13fbc8bc138836b6 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Sun, 30 Oct 2022 18:55:22 +0800 Subject: [PATCH 001/165] test: enh crash_gen function and let it run --- tests/pytest/auto_crash_gen.py | 365 ++++++++++++++++ tests/pytest/auto_crash_gen_valgrind.py | 399 ++++++++++++++++++ .../pytest/auto_crash_gen_valgrind_cluster.py | 399 ++++++++++++++++++ tests/pytest/auto_run_regular.sh | 11 + tests/pytest/auto_run_valgrind.sh | 11 + tests/pytest/auto_run_valgrind_cluster.sh | 11 + 6 files changed, 1196 insertions(+) create mode 100755 tests/pytest/auto_crash_gen.py create mode 100755 tests/pytest/auto_crash_gen_valgrind.py create mode 100755 tests/pytest/auto_crash_gen_valgrind_cluster.py create mode 100755 tests/pytest/auto_run_regular.sh create mode 100755 tests/pytest/auto_run_valgrind.sh create mode 100755 tests/pytest/auto_run_valgrind_cluster.sh diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py new file mode 100755 index 0000000000..02cca810a7 --- /dev/null +++ b/tests/pytest/auto_crash_gen.py @@ -0,0 +1,365 @@ +import os +import socket +import requests + +# -*- coding: utf-8 -*- +import os ,sys +import random +import argparse +import subprocess +import time +import platform + +# valgrind mode ? +valgrind_mode = False + +msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" } + +# formal +hostname = socket.gethostname() + +group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9' + +def get_msg(text): + return { + "msg_type": "post", + "content": { + "post": { + "zh_cn": { + "title": "Crash_gen Monitor", + "content": [ + [{ + "tag": "text", + "text": text + } + ]] + } + } + } + } + + +def send_msg(json): + headers = { + 'Content-Type': 'application/json' + } + + req = requests.post(url=group_url, headers=headers, json=json) + inf = req.json() + if "StatusCode" in inf and inf["StatusCode"] == 0: + pass + else: + print(inf) + + +# set path about run instance + +core_path = subprocess.Popen("cat /proc/sys/kernel/core_pattern", shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") +core_path = "/".join(core_path.split("/")[:-1]) +print(" ======= core path is %s ======== " %core_path) +if not os.path.exists(core_path): + os.mkdir(core_path) + +base_dir = os.path.dirname(os.path.realpath(__file__)) +if base_dir.find("community")>0: + repo = "community" +elif base_dir.find("TDengine")>0: + repo = "TDengine" +else: + repo ="TDengine" +print("base_dir:",base_dir) +home_dir = base_dir[:base_dir.find(repo)] +print("home_dir:",home_dir) +run_dir = os.path.join(home_dir,'run_dir') +run_dir = os.path.abspath(run_dir) +print("run dir is *** :",run_dir) +if not os.path.exists(run_dir): + os.mkdir(run_dir) +run_log_file = run_dir+'/crash_gen_run.log' +crash_gen_cmds_file = os.path.join(run_dir, 'crash_gen_cmds.sh') +exit_status_logs = os.path.join(run_dir, 'crash_exit.log') + +def get_path(): + buildPath='' + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + +# generate crash_gen start script randomly + +def random_args(args_list): + nums_args_list = ["--max-dbs","--num-replicas","--num-dnodes","--max-steps","--num-threads",] # record int type arguments + bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names", + "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception" + ] # record bool type arguments + strs_args_list = ["--connector-type"] # record str type arguments + + args_list["--auto-start-service"]= False + args_list["--continue-on-exception"]=True + # connect_types=['native','rest','mixed'] # restful interface has change ,we should trans dbnames to connection or change sql such as "db.test" + connect_types=['native'] + # args_list["--connector-type"]=connect_types[random.randint(0,2)] + args_list["--connector-type"]= connect_types[0] + args_list["--max-dbs"]= random.randint(1,10) + + # dnodes = [1,3] # set single dnodes; + + # args_list["--num-dnodes"]= random.sample(dnodes,1)[0] + # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"]) + args_list["--debug"]=False + args_list["--per-thread-db-connection"]=True + args_list["--track-memory-leaks"]=False + + args_list["--max-steps"]=random.randint(500,2000) + + # args_list["--ignore-errors"]=[] ## can add error codes for detail + + + args_list["--run-tdengine"]= False + args_list["--use-shadow-db"]= False + args_list["--dynamic-db-table-names"]= True + args_list["--verify-data"]= False + args_list["--record-ops"] = False + + for key in bools_args_list: + set_bool_value = [True,False] + if key == "--auto-start-service" : + continue + elif key =="--run-tdengine": + continue + elif key == "--ignore-errors": + continue + elif key == "--debug": + continue + elif key == "--per-thread-db-connection": + continue + elif key == "--continue-on-exception": + continue + elif key == "--use-shadow-db": + continue + elif key =="--track-memory-leaks": + continue + elif key == "--dynamic-db-table-names": + continue + elif key == "--verify-data": + continue + elif key == "--record-ops": + continue + else: + args_list[key]=set_bool_value[random.randint(0,1)] + + if args_list["--larger-data"]: + threads = [16,32] + else: + threads = [32,64,128,256] + args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug + + return args_list + +def limits(args_list): + if args_list["--use-shadow-db"]==True: + if args_list["--max-dbs"] > 1: + print("Cannot combine use-shadow-db with max-dbs of more than 1 ,set max-dbs=1") + args_list["--max-dbs"]=1 + else: + pass + + # env is start by test frame , not crash_gen instance + + # elif args_list["--num-replicas"]==0: + # print(" make sure num-replicas is at least 1 ") + # args_list["--num-replicas"]=1 + # elif args_list["--num-replicas"]==1: + # pass + + # elif args_list["--num-replicas"]>1: + # if not args_list["--auto-start-service"]: + # print("it should be deployed by crash_gen auto-start-service for multi replicas") + + # else: + # pass + + return args_list + +def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + + bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names", + "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"] + arguments = "" + for k ,v in args_list.items(): + if k == "--ignore-errors": + if v: + arguments+=(k+"="+str(v)+" ") + else: + arguments+="" + elif k in bools_args_list and v==True: + arguments+=(k+" ") + elif k in bools_args_list and v==False: + arguments+="" + else: + arguments+=(k+"="+str(v)+" ") + + if valgrind : + + crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path ,arguments) + + else: + + crash_gen_cmd = 'cd %s && ./crash_gen.sh %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550'%(crash_gen_path ,arguments) + + return crash_gen_cmd + +def start_taosd(): + build_path = get_path() + if repo == "community": + start_path = build_path[:-5]+"community/tests/system-test/" + elif repo == "TDengine": + start_path = build_path[:-5]+"/tests/system-test/" + else: + pass + + start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path) + os.system(start_cmd) + +def get_cmds(args_list): + # build_path = get_path() + # if repo == "community": + # crash_gen_path = build_path[:-5]+"community/tests/pytest/" + # elif repo == "TDengine": + # crash_gen_path = build_path[:-5]+"/tests/pytest/" + # else: + # pass + + # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path) + + crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) + return crash_gen_cmd + +def run_crash_gen(crash_cmds): + + # prepare env of taosd + start_taosd() + + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + result_file = os.path.join(crash_gen_path, 'valgrind.out') + + + # run crash_gen and back logs + os.system('echo "%s">>%s'%(crash_cmds,crash_gen_cmds_file)) + os.system("%s >>%s "%(crash_cmds,result_file)) + + +def check_status(): + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + result_file = os.path.join(crash_gen_path, 'valgrind.out') + run_code = subprocess.Popen("tail -n 50 %s"%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs)) + + core_check = subprocess.Popen('ls -l %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if int(core_check.strip().rstrip()) > 0: + # it means core files has occured + return 3 + + if "Crash_Gen is now exiting with status code: 1" in run_code: + return 1 + elif "Crash_Gen is now exiting with status code: 0" in run_code: + return 0 + else: + return 2 + + +def main(): + + args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[], + "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False, + "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , + "--continue-on-exception":False } + + args = random_args(args_list) + args = limits(args) + + + build_path = get_path() + os.system("pip install git+https://github.com/taosdata/taos-connector-python.git") + if repo =="community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo =="TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + + if os.path.exists(crash_gen_path+"crash_gen.sh"): + print(" make sure crash_gen.sh is ready") + else: + print( " crash_gen.sh is not exists ") + sys.exit(1) + + git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16] + + # crash_cmds = get_cmds() + + crash_cmds = get_cmds(args) + # clean run_dir + os.system('rm -rf %s'%run_dir ) + if not os.path.exists(run_dir): + os.mkdir(run_dir) + print(crash_cmds) + run_crash_gen(crash_cmds) + status = check_status() + + print("exit status : ", status) + + if status ==4: + print('======== crash_gen found memory bugs ========') + if status ==5: + print('======== crash_gen found memory errors ========') + if status >0: + print('======== crash_gen run failed and not exit as expected ========') + else: + print('======== crash_gen run sucess and exit as expected ========') + + + if status!=0 : + + try: + text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + send_msg(get_msg(text)) + except Exception as e: + print("exception:", e) + exit(status) + + +if __name__ == '__main__': + main() + + diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py new file mode 100755 index 0000000000..1443dcd543 --- /dev/null +++ b/tests/pytest/auto_crash_gen_valgrind.py @@ -0,0 +1,399 @@ +#!/usr/bin/python3 + + +import os +import socket +import requests + +# -*- coding: utf-8 -*- +import os ,sys +import random +import argparse +import subprocess +import time +import platform + +# valgrind mode ? +valgrind_mode = True + +msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" } + +# formal +hostname = socket.gethostname() + +group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9' + +def get_msg(text): + return { + "msg_type": "post", + "content": { + "post": { + "zh_cn": { + "title": "Crash_gen Monitor", + "content": [ + [{ + "tag": "text", + "text": text + } + ]] + } + } + } + } + + +def send_msg(json): + headers = { + 'Content-Type': 'application/json' + } + + req = requests.post(url=group_url, headers=headers, json=json) + inf = req.json() + if "StatusCode" in inf and inf["StatusCode"] == 0: + pass + else: + print(inf) + + +# set path about run instance + +core_path = subprocess.Popen("cat /proc/sys/kernel/core_pattern", shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") +core_path = "/".join(core_path.split("/")[:-1]) +print(" ======= core path is %s ======== " %core_path) +if not os.path.exists(core_path): + os.mkdir(core_path) + +base_dir = os.path.dirname(os.path.realpath(__file__)) +if base_dir.find("community")>0: + repo = "community" +elif base_dir.find("TDengine")>0: + repo = "TDengine" +else: + repo ="TDengine" +print("base_dir:",base_dir) +home_dir = base_dir[:base_dir.find(repo)] +print("home_dir:",home_dir) +run_dir = os.path.join(home_dir,'run_dir') +run_dir = os.path.abspath(run_dir) +print("run dir is *** :",run_dir) +if not os.path.exists(run_dir): + os.mkdir(run_dir) +run_log_file = run_dir+'/crash_gen_run.log' +crash_gen_cmds_file = os.path.join(run_dir, 'crash_gen_cmds.sh') +exit_status_logs = os.path.join(run_dir, 'crash_exit.log') + +def get_path(): + buildPath='' + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + +# generate crash_gen start script randomly + +def random_args(args_list): + nums_args_list = ["--max-dbs","--num-replicas","--num-dnodes","--max-steps","--num-threads",] # record int type arguments + bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names", + "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception" + ] # record bool type arguments + strs_args_list = ["--connector-type"] # record str type arguments + + args_list["--auto-start-service"]= False + args_list["--continue-on-exception"]=True + # connect_types=['native','rest','mixed'] # restful interface has change ,we should trans dbnames to connection or change sql such as "db.test" + connect_types=['native'] + # args_list["--connector-type"]=connect_types[random.randint(0,2)] + args_list["--connector-type"]= connect_types[0] + args_list["--max-dbs"]= random.randint(1,10) + + # dnodes = [1,3] # set single dnodes; + + # args_list["--num-dnodes"]= random.sample(dnodes,1)[0] + # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"]) + args_list["--debug"]=False + args_list["--per-thread-db-connection"]=True + args_list["--track-memory-leaks"]=False + + args_list["--max-steps"]=random.randint(200,500) + + threads = [16,32] + + args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug + # args_list["--ignore-errors"]=[] ## can add error codes for detail + + + args_list["--run-tdengine"]= False + args_list["--use-shadow-db"]= False + args_list["--dynamic-db-table-names"]= True + args_list["--verify-data"]= False + args_list["--record-ops"] = False + + for key in bools_args_list: + set_bool_value = [True,False] + if key == "--auto-start-service" : + continue + elif key =="--run-tdengine": + continue + elif key == "--ignore-errors": + continue + elif key == "--debug": + continue + elif key == "--per-thread-db-connection": + continue + elif key == "--continue-on-exception": + continue + elif key == "--use-shadow-db": + continue + elif key =="--track-memory-leaks": + continue + elif key == "--dynamic-db-table-names": + continue + elif key == "--verify-data": + continue + elif key == "--record-ops": + continue + elif key == "--larger-data": + continue + else: + args_list[key]=set_bool_value[random.randint(0,1)] + return args_list + +def limits(args_list): + if args_list["--use-shadow-db"]==True: + if args_list["--max-dbs"] > 1: + print("Cannot combine use-shadow-db with max-dbs of more than 1 ,set max-dbs=1") + args_list["--max-dbs"]=1 + else: + pass + + # env is start by test frame , not crash_gen instance + + # elif args_list["--num-replicas"]==0: + # print(" make sure num-replicas is at least 1 ") + # args_list["--num-replicas"]=1 + # elif args_list["--num-replicas"]==1: + # pass + + # elif args_list["--num-replicas"]>1: + # if not args_list["--auto-start-service"]: + # print("it should be deployed by crash_gen auto-start-service for multi replicas") + + # else: + # pass + + return args_list + +def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + + bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names", + "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"] + arguments = "" + for k ,v in args_list.items(): + if k == "--ignore-errors": + if v: + arguments+=(k+"="+str(v)+" ") + else: + arguments+="" + elif k in bools_args_list and v==True: + arguments+=(k+" ") + elif k in bools_args_list and v==False: + arguments+="" + else: + arguments+=(k+"="+str(v)+" ") + + if valgrind : + + crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path ,arguments) + + else: + + crash_gen_cmd = 'cd %s && ./crash_gen.sh %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550'%(crash_gen_path ,arguments) + + return crash_gen_cmd + + +def start_taosd(): + build_path = get_path() + if repo == "community": + start_path = build_path[:-5]+"community/tests/system-test/" + elif repo == "TDengine": + start_path = build_path[:-5]+"/tests/system-test/" + else: + pass + + start_cmd = 'cd %s && python3 test.py '%(start_path) + os.system(start_cmd +">>/dev/null") + +def get_cmds(args_list): + # build_path = get_path() + # if repo == "community": + # crash_gen_path = build_path[:-5]+"community/tests/pytest/" + # elif repo == "TDengine": + # crash_gen_path = build_path[:-5]+"/tests/pytest/" + # else: + # pass + + # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path) + + crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) + return crash_gen_cmd + +def run_crash_gen(crash_cmds): + + # prepare env of taosd + start_taosd() + # run crash_gen and back logs + os.system('echo "%s">>%s'%(crash_cmds,crash_gen_cmds_file)) + # os.system("cp %s %s"%(crash_gen_cmds_file, core_path)) + os.system("%s"%(crash_cmds)) + +def check_status(): + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + result_file = os.path.join(crash_gen_path, 'valgrind.out') + run_code = subprocess.Popen("tail -n 50 %s"%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs)) + + core_check = subprocess.Popen('ls -l %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if int(core_check.strip().rstrip()) > 0: + # it means core files has occured + return 3 + + mem_status = check_memory() + if mem_status >0: + return mem_status + if "Crash_Gen is now exiting with status code: 1" in run_code: + return 1 + elif "Crash_Gen is now exiting with status code: 0" in run_code: + return 0 + else: + return 2 + + +def check_memory(): + + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + ''' + invalid read, invalid write + ''' + back_path = os.path.join(core_path,"valgrind_report") + if not os.path.exists(back_path): + os.mkdir(back_path) + + stderr_file = os.path.join(crash_gen_path , "valgrind.err") + + status = 0 + + grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if grep_res: + # os.system("cp %s %s"%(stderr_file , back_path)) + status = 4 + + grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if grep_res: + # os.system("cp %s %s"%(stderr_file , back_path)) + status = 4 + + grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if grep_res: + # os.system("cp %s %s"%(stderr_file , back_path)) + status = 5 + + return status + +def main(): + + args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[], + "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False, + "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , + "--continue-on-exception":False } + + args = random_args(args_list) + args = limits(args) + + build_path = get_path() + os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null") + if repo =="community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo =="TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + + if os.path.exists(crash_gen_path+"crash_gen.sh"): + print(" make sure crash_gen.sh is ready") + else: + print( " crash_gen.sh is not exists ") + sys.exit(1) + + git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16] + + # crash_cmds = get_cmds() + + crash_cmds = get_cmds(args) + + # clean run_dir + os.system('rm -rf %s'%run_dir ) + if not os.path.exists(run_dir): + os.mkdir(run_dir) + print(crash_cmds) + run_crash_gen(crash_cmds) + status = check_status() + # back_path = os.path.join(core_path,"valgrind_report") + + print("exit status : ", status) + + if status ==4: + print('======== crash_gen found memory bugs ========') + if status ==5: + print('======== crash_gen found memory errors ========') + if status >0: + print('======== crash_gen run failed and not exit as expected ========') + else: + print('======== crash_gen run sucess and exit as expected ========') + + if status!=0 : + + try: + text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + send_msg(get_msg(text)) + except Exception as e: + print("exception:", e) + exit(status) + + +if __name__ == '__main__': + main() + + diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py new file mode 100755 index 0000000000..05cdaa6cc5 --- /dev/null +++ b/tests/pytest/auto_crash_gen_valgrind_cluster.py @@ -0,0 +1,399 @@ +#!/usr/bin/python3 + + +import os +import socket +import requests + +# -*- coding: utf-8 -*- +import os ,sys +import random +import argparse +import subprocess +import time +import platform + +# valgrind mode ? +valgrind_mode = True + +msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" } + +# formal +hostname = socket.gethostname() + +group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9' + +def get_msg(text): + return { + "msg_type": "post", + "content": { + "post": { + "zh_cn": { + "title": "Crash_gen Monitor", + "content": [ + [{ + "tag": "text", + "text": text + } + ]] + } + } + } + } + + +def send_msg(json): + headers = { + 'Content-Type': 'application/json' + } + + req = requests.post(url=group_url, headers=headers, json=json) + inf = req.json() + if "StatusCode" in inf and inf["StatusCode"] == 0: + pass + else: + print(inf) + + +# set path about run instance + +core_path = subprocess.Popen("cat /proc/sys/kernel/core_pattern", shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") +core_path = "/".join(core_path.split("/")[:-1]) +print(" ======= core path is %s ======== " %core_path) +if not os.path.exists(core_path): + os.mkdir(core_path) + +base_dir = os.path.dirname(os.path.realpath(__file__)) +if base_dir.find("community")>0: + repo = "community" +elif base_dir.find("TDengine")>0: + repo = "TDengine" +else: + repo ="TDengine" +print("base_dir:",base_dir) +home_dir = base_dir[:base_dir.find(repo)] +print("home_dir:",home_dir) +run_dir = os.path.join(home_dir,'run_dir') +run_dir = os.path.abspath(run_dir) +print("run dir is *** :",run_dir) +if not os.path.exists(run_dir): + os.mkdir(run_dir) +run_log_file = run_dir+'/crash_gen_run.log' +crash_gen_cmds_file = os.path.join(run_dir, 'crash_gen_cmds.sh') +exit_status_logs = os.path.join(run_dir, 'crash_exit.log') + +def get_path(): + buildPath='' + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + +# generate crash_gen start script randomly + +def random_args(args_list): + nums_args_list = ["--max-dbs","--num-replicas","--num-dnodes","--max-steps","--num-threads",] # record int type arguments + bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names", + "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception" + ] # record bool type arguments + strs_args_list = ["--connector-type"] # record str type arguments + + args_list["--auto-start-service"]= False + args_list["--continue-on-exception"]=True + # connect_types=['native','rest','mixed'] # restful interface has change ,we should trans dbnames to connection or change sql such as "db.test" + connect_types=['native'] + # args_list["--connector-type"]=connect_types[random.randint(0,2)] + args_list["--connector-type"]= connect_types[0] + args_list["--max-dbs"]= random.randint(1,10) + + # dnodes = [1,3] # set single dnodes; + + # args_list["--num-dnodes"]= random.sample(dnodes,1)[0] + # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"]) + args_list["--debug"]=False + args_list["--per-thread-db-connection"]=True + args_list["--track-memory-leaks"]=False + + args_list["--max-steps"]=random.randint(200,500) + + threads = [16,32] + + args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug + # args_list["--ignore-errors"]=[] ## can add error codes for detail + + + args_list["--run-tdengine"]= False + args_list["--use-shadow-db"]= False + args_list["--dynamic-db-table-names"]= True + args_list["--verify-data"]= False + args_list["--record-ops"] = False + + for key in bools_args_list: + set_bool_value = [True,False] + if key == "--auto-start-service" : + continue + elif key =="--run-tdengine": + continue + elif key == "--ignore-errors": + continue + elif key == "--debug": + continue + elif key == "--per-thread-db-connection": + continue + elif key == "--continue-on-exception": + continue + elif key == "--use-shadow-db": + continue + elif key =="--track-memory-leaks": + continue + elif key == "--dynamic-db-table-names": + continue + elif key == "--verify-data": + continue + elif key == "--record-ops": + continue + elif key == "--larger-data": + continue + else: + args_list[key]=set_bool_value[random.randint(0,1)] + return args_list + +def limits(args_list): + if args_list["--use-shadow-db"]==True: + if args_list["--max-dbs"] > 1: + print("Cannot combine use-shadow-db with max-dbs of more than 1 ,set max-dbs=1") + args_list["--max-dbs"]=1 + else: + pass + + # env is start by test frame , not crash_gen instance + + # elif args_list["--num-replicas"]==0: + # print(" make sure num-replicas is at least 1 ") + # args_list["--num-replicas"]=1 + # elif args_list["--num-replicas"]==1: + # pass + + # elif args_list["--num-replicas"]>1: + # if not args_list["--auto-start-service"]: + # print("it should be deployed by crash_gen auto-start-service for multi replicas") + + # else: + # pass + + return args_list + +def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + + bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names", + "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"] + arguments = "" + for k ,v in args_list.items(): + if k == "--ignore-errors": + if v: + arguments+=(k+"="+str(v)+" ") + else: + arguments+="" + elif k in bools_args_list and v==True: + arguments+=(k+" ") + elif k in bools_args_list and v==False: + arguments+="" + else: + arguments+=(k+"="+str(v)+" ") + + if valgrind : + + crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0707 '%(crash_gen_path ,arguments) + + else: + + crash_gen_cmd = 'cd %s && ./crash_gen.sh -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0014,0x0707'%(crash_gen_path ,arguments) + + return crash_gen_cmd + + +def start_taosd(): + build_path = get_path() + if repo == "community": + start_path = build_path[:-5]+"community/tests/system-test/" + elif repo == "TDengine": + start_path = build_path[:-5]+"/tests/system-test/" + else: + pass + + start_cmd = 'cd %s && python3 test.py -N 4 -M 1 '%(start_path) + os.system(start_cmd +">>/dev/null") + +def get_cmds(args_list): + # build_path = get_path() + # if repo == "community": + # crash_gen_path = build_path[:-5]+"community/tests/pytest/" + # elif repo == "TDengine": + # crash_gen_path = build_path[:-5]+"/tests/pytest/" + # else: + # pass + + # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path) + + crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) + return crash_gen_cmd + +def run_crash_gen(crash_cmds): + + # prepare env of taosd + start_taosd() + # run crash_gen and back logs + os.system('echo "%s">>%s'%(crash_cmds,crash_gen_cmds_file)) + # os.system("cp %s %s"%(crash_gen_cmds_file, core_path)) + os.system("%s"%(crash_cmds)) + +def check_status(): + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + result_file = os.path.join(crash_gen_path, 'valgrind.out') + run_code = subprocess.Popen("tail -n 50 %s"%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs)) + + core_check = subprocess.Popen('ls -l %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if int(core_check.strip().rstrip()) > 0: + # it means core files has occured + return 3 + + mem_status = check_memory() + if mem_status >0: + return mem_status + if "Crash_Gen is now exiting with status code: 1" in run_code: + return 1 + elif "Crash_Gen is now exiting with status code: 0" in run_code: + return 0 + else: + return 2 + + +def check_memory(): + + build_path = get_path() + if repo == "community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo == "TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + ''' + invalid read, invalid write + ''' + back_path = os.path.join(core_path,"valgrind_report") + if not os.path.exists(back_path): + os.mkdir(back_path) + + stderr_file = os.path.join(crash_gen_path , "valgrind.err") + + status = 0 + + grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if grep_res: + # os.system("cp %s %s"%(stderr_file , back_path)) + status = 4 + + grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if grep_res: + # os.system("cp %s %s"%(stderr_file , back_path)) + status = 4 + + grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + + if grep_res: + # os.system("cp %s %s"%(stderr_file , back_path)) + status = 5 + + return status + +def main(): + + args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[], + "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False, + "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , + "--continue-on-exception":False } + + args = random_args(args_list) + args = limits(args) + + build_path = get_path() + os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null") + if repo =="community": + crash_gen_path = build_path[:-5]+"community/tests/pytest/" + elif repo =="TDengine": + crash_gen_path = build_path[:-5]+"/tests/pytest/" + else: + pass + + if os.path.exists(crash_gen_path+"crash_gen.sh"): + print(" make sure crash_gen.sh is ready") + else: + print( " crash_gen.sh is not exists ") + sys.exit(1) + + git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16] + + # crash_cmds = get_cmds() + + crash_cmds = get_cmds(args) + + # clean run_dir + os.system('rm -rf %s'%run_dir ) + if not os.path.exists(run_dir): + os.mkdir(run_dir) + print(crash_cmds) + run_crash_gen(crash_cmds) + status = check_status() + # back_path = os.path.join(core_path,"valgrind_report") + + print("exit status : ", status) + + if status ==4: + print('======== crash_gen found memory bugs ========') + if status ==5: + print('======== crash_gen found memory errors ========') + if status >0: + print('======== crash_gen run failed and not exit as expected ========') + else: + print('======== crash_gen run sucess and exit as expected ========') + + if status!=0 : + + try: + text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + send_msg(get_msg(text)) + except Exception as e: + print("exception:", e) + exit(status) + + +if __name__ == '__main__': + main() + + diff --git a/tests/pytest/auto_run_regular.sh b/tests/pytest/auto_run_regular.sh new file mode 100755 index 0000000000..27e8013269 --- /dev/null +++ b/tests/pytest/auto_run_regular.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# set LD_LIBRARY_PATH +export PATH=$PATH:/home/TDengine/debug/build/bin +export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib +ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null +ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null +ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null + +# run crash_gen auto script +python3 /home/TDengine/tests/pytest/auto_crash_gen.py \ No newline at end of file diff --git a/tests/pytest/auto_run_valgrind.sh b/tests/pytest/auto_run_valgrind.sh new file mode 100755 index 0000000000..c7154e867c --- /dev/null +++ b/tests/pytest/auto_run_valgrind.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# set LD_LIBRARY_PATH +export PATH=$PATH:/home/TDengine/debug/build/bin +export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib +ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null +ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null +ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null + +# run crash_gen auto script +python3 /home/TDengine/tests/pytest/auto_crash_gen_valgrind.py \ No newline at end of file diff --git a/tests/pytest/auto_run_valgrind_cluster.sh b/tests/pytest/auto_run_valgrind_cluster.sh new file mode 100755 index 0000000000..62bc22e923 --- /dev/null +++ b/tests/pytest/auto_run_valgrind_cluster.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# set LD_LIBRARY_PATH +export PATH=$PATH:/home/TDengine/debug/build/bin +export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib +ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null +ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null +ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null + +# run crash_gen auto script +python3 /home/TDengine/tests/pytest/auto_crash_gen_valgrind_cluster.py \ No newline at end of file From 17a502f569353e8a7c2f66632f59502f50beb1bd Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 7 Nov 2022 13:44:12 +0800 Subject: [PATCH 002/165] update --- tests/pytest/crash_gen.sh | 2 +- tests/pytest/crash_gen/crash_gen_main.py | 478 +++++++++++++++++++++-- 2 files changed, 456 insertions(+), 24 deletions(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 539314dea4..cc2941a52a 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -45,7 +45,7 @@ fi # Now getting ready to execute Python # The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk -PYTHON_EXEC=python3.8 +PYTHON_EXEC=python3 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. # export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 600c64b8e6..d8946780a2 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -254,7 +254,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 120 # Normal: 120 + WORKER_THREAD_TIMEOUT = 1200 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -674,9 +674,12 @@ class AnyState: # only "under normal circumstances", as we may override it with the -b option CAN_DROP_DB = 2 CAN_CREATE_FIXED_SUPER_TABLE = 3 + CAN_CREATE_STREAM = 3 # super table must exists CAN_DROP_FIXED_SUPER_TABLE = 4 CAN_ADD_DATA = 5 + CAN_DROP_STREAM = 5 CAN_READ_DATA = 6 + CAN_DELETE_DATA = 6 def __init__(self): self._info = self.getInfo() @@ -727,12 +730,18 @@ class AnyState: return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] + def canCreateStream(self): + return self._info[self.CAN_CREATE_STREAM] + def canAddData(self): return self._info[self.CAN_ADD_DATA] def canReadData(self): return self._info[self.CAN_READ_DATA] + def canDeleteData(self): + return self._info[self.CAN_DELETE_DATA] + def assertAtMostOneSuccess(self, tasks, cls): sCnt = 0 for task in tasks: @@ -921,7 +930,7 @@ class StateMechine: except taos.error.ProgrammingError as err: Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) traceback.print_stack() - raise # re-throw + pass # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -974,14 +983,21 @@ class StateMechine: # did not do this when openning connection, and this is NOT the worker # thread, which does this on their own dbc.use(dbName) + if not dbc.hasTables(): # no tables + Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) return StateDbOnly() # For sure we have tables, which means we must have the super table. # TODO: are we sure? + sTable = self._db.getFixedSuperTable() - if sTable.hasRegTables(dbc): # no regular tables + + + if sTable.hasRegTables(dbc): # no regular tables + # print("debug=====*\n"*100) Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) + return StateSuperTableOnly() else: # has actual tables Logging.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) @@ -1109,6 +1125,7 @@ class Database: return "fs_table" def getFixedSuperTable(self) -> TdSuperTable: + return TdSuperTable(self.getFixedSuperTableName(), self.getName()) # We aim to create a starting time tick, such that, whenever we run our test here once @@ -1342,6 +1359,11 @@ class Task(): 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched 0x2662, # Table does not exist #TODO: what about 2603 above? + 0x032C, # Object is creating + 0x032D, # Object is dropping + 0x03D3, # Conflict transaction not completed + 0x0707, # Query not ready , it always occur at replica 3 + 0x707, # Query not ready @@ -1638,9 +1660,12 @@ class TaskCreateDb(StateTransitionTask): # numReplica = Dice.throw(Settings.getConfig().max_replicas) + 1 # 1,2 ... N numReplica = Config.getConfig().num_replicas # fixed, always repStr = "replica {}".format(numReplica) - updatePostfix = "update 1" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active + updatePostfix = "" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active , 3.0 version default is update 1 + vg_nums = random.randint(1,8) + cache_model = Dice.choice(['none' , 'last_row' , 'last_value' , 'both']) + buffer = random.randint(3,128) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) ) + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model,buffer ) ) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) ) @@ -1657,6 +1682,69 @@ class TaskDropDb(StateTransitionTask): self.execWtSql(wt, "drop database {}".format(self._db.getName())) Logging.debug("[OPS] database dropped at {}".format(time.time())) +class TaskCreateStream(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sub_stream_name = dbname+ '_sub_stream' + sub_stream_tb_name = 'avg_sub' + super_stream_name = dbname+ '_super_stream' + super_stream_tb_name = 'avg_super' + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create stream + ''' + CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + ''' + stbname =sTable.getName() + sub_tables = sTable.getRegTables(wt.getDbConn()) + if sub_tables: + + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + + # create stream with query above sub_table + stream_sql = 'create stream {} into {}.{} as select count(*), avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,dbname,sub_tbname) + try: + self.execWtSql(wt, stream_sql) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # stream already exists + # stream need drop before drop table + pass + + sTable.setStreamName(sub_stream_name) + else: + pass + + else: + stream_sql = 'create stream {} into {}.{} as select count(*), avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,dbname,stbname) + + try: + self.execWtSql(wt, stream_sql) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # stream already exists + # stream need drop before drop table + pass + + + + class TaskCreateSuperTable(StateTransitionTask): @classmethod def getEndState(cls): @@ -1688,15 +1776,40 @@ class TdSuperTable: def __init__(self, stName, dbName): self._stName = stName self._dbName = dbName + self._streamName = [] def getName(self): return self._stName + def setStreamName(self,name): + self._streamName.append(name) + + def getStreamName(self): + return self._streamName + def drop(self, dbc, skipCheck = False): dbName = self._dbName if self.exists(dbc) : # if myself exists - fullTableName = dbName + '.' + self._stName - dbc.execute("DROP TABLE {}".format(fullTableName)) + fullTableName = dbName + '.' + self._stName + try: + dbc.execute("DROP TABLE {}".format(fullTableName)) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist + pass + # # stream need drop before drop table + # for stream in self.getStreamName(): + # drop_stream_sql = 'drop stream {}'.format(stream) + # try: + # dbc.execute(drop_stream_sql) + # except taos.error.ProgrammingError as err: + # # correcting for strange error number scheme + # errno3 = Helper.convertErrno(err.errno) + # if errno3 in [1011,0x3F3,0x03f3,0x2662,0x03f1]: # stream not exists + # pass + # dbc.execute("DROP TABLE {}".format(fullTableName)) + # pass + else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) @@ -1711,10 +1824,17 @@ class TdSuperTable: dbName = self._dbName dbc.execute("USE " + dbName) - fullTableName = dbName + '.' + self._stName + fullTableName = dbName + '.' + self._stName + if dbc.existsSuperTable(self._stName): if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + + pass + + + # dbc.execute("DROP TABLE {}".format(fullTableName)) else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -1733,7 +1853,7 @@ class TdSuperTable: def getRegTables(self, dbc: DbConn): dbName = self._dbName try: - dbc.query("select TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later + dbc.query("select distinct TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) Logging.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) @@ -1743,7 +1863,44 @@ class TdSuperTable: return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation def hasRegTables(self, dbc: DbConn): - return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 + # print(self._stName) + # dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) + # print(dbc.getQueryResult()) + if self.hasStreamTables(dbc) or self.hasStreams(dbc): + if self.dropStreams(dbc): + self.dropStreamTables(dbc) + if dbc.existsSuperTable(self._stName): + + return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 + else: + return False + + def hasStreamTables(self,dbc: DbConn): + + return dbc.query("show {}.stables like 'avg%'".format(self._dbName)) > 0 + + def hasStreams(self,dbc: DbConn): + return dbc.query("show streams") > 0 + + def dropStreams(self,dbc:DbConn): + dbc.query("show streams ") + Streams = dbc.getQueryResult() + for Stream in Streams: + if Stream[0].startswith(self._dbName): + dbc.execute('drop stream {}'.format(Stream[0])) + + return not dbc.query("show streams ") > 0 + + def dropStreamTables(self, dbc: DbConn): + dbc.query("show {}.stables like 'avg%'".format(self._dbName)) + + StreamTables = dbc.getQueryResult() + + for StreamTable in StreamTables: + if self.dropStreams: + dbc.execute('drop table {}.{}'.format(self._dbName,StreamTable[0])) + + return not dbc.query("show {}.stables like 'avg%'".format(self._dbName)) def ensureRegTable(self, task: Optional[Task], dbc: DbConn, regTableName: str): ''' @@ -1838,10 +1995,46 @@ class TdSuperTable: # Run the query against the regular table first doAggr = (Dice.throw(2) == 0) # 1 in 2 chance if not doAggr: # don't do aggregate query, just simple one + commonExpr = Dice.choice([ + '*', + # 'abs(speed)', + # 'acos(speed)', + # 'asin(speed)', + # 'atan(speed)', + # 'ceil(speed)', + # 'cos(speed)', + # 'cos(speed)', + # 'floor(speed)', + # 'log(speed,2)', + # 'pow(speed,2)', + # 'round(speed)', + # 'sin(speed)', + # 'sqrt(speed)', + # 'char_length(color)', + # 'concat(color,color)', + # 'concat_ws(" ", color,color," ")', + # 'length(color)', + # 'lower(color)', + # 'ltrim(color)', + # 'substr(color , 2)', + # 'upper(color)', + # 'cast(speed as double)', + # 'cast(ts as bigint)', + # # 'TO_ISO8601(color)', + # # 'TO_UNIXTIMESTAMP(ts)', + # 'now()', + # 'timediff(ts,now)', + # 'timezone()', + # 'TIMETRUNCATE(ts,1s)', + # 'TIMEZONE()', + # 'TODAY()', + # 'distinct(color)' + ] + ) ret.append(SqlQuery( # reg table - "select {} from {}.{}".format('*', self._dbName, rTbName))) + "select {} from {}.{}".format(commonExpr, self._dbName, rTbName))) ret.append(SqlQuery( # super table - "select {} from {}.{}".format('*', self._dbName, self.getName()))) + "select {} from {}.{}".format(commonExpr, self._dbName, self.getName()))) else: # Aggregate query aggExpr = Dice.choice([ 'count(*)', @@ -1857,17 +2050,38 @@ class TdSuperTable: 'top(speed, 50)', # TODO: not supported? 'bottom(speed, 50)', # TODO: not supported? 'apercentile(speed, 10)', # TODO: TD-1316 - # 'last_row(speed)', # TODO: commented out per TD-3231, we should re-create + 'last_row(*)', # TODO: commented out per TD-3231, we should re-create # Transformation Functions # 'diff(speed)', # TODO: no supported?! - 'spread(speed)' + 'spread(speed)', + # 'elapsed(ts)', + # 'mode(speed)', + # 'bottom(speed,1)', + # 'top(speed,1)', + # 'tail(speed,1)', + # 'unique(color)', + # 'csum(speed)', + # 'DERIVATIVE(speed,1s,1)', + # 'diff(speed,1)', + # 'irate(speed)', + # 'mavg(speed,3)', + # 'sample(speed,5)', + # 'STATECOUNT(speed,"LT",1)', + # 'STATEDURATION(speed,"LT",1)', + # 'twa(speed)' + + + + + ]) # TODO: add more from 'top' # if aggExpr not in ['stddev(speed)']: # STDDEV not valid for super tables?! (Done in TD-1049) sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName()) if Dice.throw(3) == 0: # 1 in X chance - sql = sql + ' GROUP BY color' + partion_expr = Dice.choice(['color','tbname']) + sql = sql + ' partition BY ' + partion_expr + ' order by ' + partion_expr Progress.emit(Progress.QUERY_GROUP_BY) # Logging.info("Executing GROUP-BY query: " + sql) ret.append(SqlQuery(sql)) @@ -1964,16 +2178,17 @@ class TaskDropSuperTable(StateTransitionTask): isSuccess = True for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) + streams_prix = self._db.getName() try: self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = Helper.convertErrno(err.errno) - if (errno2 in [0x362]): # mnode invalid table name - isSuccess = False - Logging.debug("[DB] Acceptable error when dropping a table") - continue # try to delete next regular table + pass + + + + + if (not tickOutput): tickOutput = True # Print only one time @@ -1984,7 +2199,17 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + try: + self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + except taos.error.ProgrammingError as err: + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) + if (errno2 in [0x362]): # mnode invalid table name + isSuccess = False + Logging.debug("[DB] Acceptable error when dropping a table") + elif errno2 in [1011,0x3F3,0x03f3]: # table doesn't exist + + pass class TaskAlterTags(StateTransitionTask): @@ -2234,6 +2459,212 @@ class TaskAddData(StateTransitionTask): self.activeTable.discard(i) # not raising an error, unlike remove +class TaskDeleteData(StateTransitionTask): + # Track which table is being actively worked on + activeTable: Set[int] = set() + + # We use these two files to record operations to DB, useful for power-off tests + fAddLogReady = None # type: Optional[io.TextIOWrapper] + fAddLogDone = None # type: Optional[io.TextIOWrapper] + + @classmethod + def prepToRecordOps(cls): + if Config.getConfig().record_ops: + if (cls.fAddLogReady is None): + Logging.info( + "Recording in a file operations to be performed...") + cls.fAddLogReady = open("add_log_ready.txt", "w") + if (cls.fAddLogDone is None): + Logging.info("Recording in a file operations completed...") + cls.fAddLogDone = open("add_log_done.txt", "w") + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDeleteData() + + def _lockTableIfNeeded(self, fullTableName, extraMsg = ''): + if Config.getConfig().verify_data: + # Logging.info("Locking table: {}".format(fullTableName)) + self.lockTable(fullTableName) + # Logging.info("Table locked {}: {}".format(extraMsg, fullTableName)) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + else: + # Logging.info("Skipping locking table") + pass + + def _unlockTableIfNeeded(self, fullTableName): + if Config.getConfig().verify_data: + # Logging.info("Unlocking table: {}".format(fullTableName)) + self.unlockTable(fullTableName) + # Logging.info("Table unlocked: {}".format(fullTableName)) + else: + pass + # Logging.info("Skipping unlocking table") + + def _deleteData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + del_Records = int(numRecords/5) + if Dice.throw(2) == 0: + for j in range(del_Records): # number of records per table + intToWrite = db.getNextInt() + nextTick = db.getNextTick() + # nextColor = db.getNextColor() + if Config.getConfig().record_ops: + self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") + self.fAddLogReady.write("Ready to delete {} to {}\n".format(intToWrite, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady.fileno()) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + self._lockTableIfNeeded(fullTableName) # so that we are verify read-back. TODO: deal with exceptions before unlock + + try: + sql = "delete from {} where ts = '{}' ;".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick) + + # print(sql) + # Logging.info("Adding data: {}".format(sql)) + dbc.execute(sql) + # Logging.info("Data added: {}".format(sql)) + intWrote = intToWrite + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + intToUpdate = db.getNextInt() # Updated， but should not succeed + # nextColor = db.getNextColor() + sql = "delete from {} where ts = '{}' ;".format( # "INSERt" means "update" here + fullTableName, + nextTick) + # sql = "UPDATE {} set speed={}, color='{}' WHERE ts='{}'".format( + # fullTableName, db.getNextInt(), db.getNextColor(), nextTick) + dbc.execute(sql) + intWrote = intToUpdate # We updated, seems TDengine non-cluster accepts this. + + except: # Any exception at all + self._unlockTableIfNeeded(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if Config.getConfig().verify_data: # only if command line asks for it + try: + readBack = dbc.queryScalar("SELECT * from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + if readBack == None : + pass + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + print("D1",end="") # D1 means delete data success and only 1 record + + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + pass + else: + # Re-throw otherwise + raise + finally: + self._unlockTableIfNeeded(fullTableName) # Quite ugly, refactor lock/unlock + # Done with read-back verification, unlock the table now + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(intWrote) + else: + + # delete all datas and verify datas ,expected table is empty + if Config.getConfig().record_ops: + self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") + self.fAddLogReady.write("Ready to delete {} to {}\n".format(intToWrite, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady.fileno()) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + self._lockTableIfNeeded(fullTableName) # so that we are verify read-back. TODO: deal with exceptions before unlock + + try: + sql = "delete from {} ;".format( # removed: tags ('{}', {}) + fullTableName) + # Logging.info("Adding data: {}".format(sql)) + dbc.execute(sql) + # Logging.info("Data added: {}".format(sql)) + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + sql = "delete from {} ;".format( # "INSERt" means "update" here + fullTableName) + dbc.execute(sql) + + except: # Any exception at all + self._unlockTableIfNeeded(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if Config.getConfig().verify_data: # only if command line asks for it + try: + readBack = dbc.queryScalar("SELECT * from {}.{} ". + format(db.getName(), regTableName)) + if readBack == None : + pass + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + print("Da",end="") # Da means delete data success and for all datas + + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + pass + else: + # Re-throw otherwise + raise + finally: + self._unlockTableIfNeeded(fullTableName) # Quite ugly, refactor lock/unlock + # Done with read-back verification, unlock the table now + + if Config.getConfig().record_ops: + if self.fAddLogDone is None: + raise CrashGenError("Unexpected empty fAddLogDone") + self.fAddLogDone.write("Wrote {} to {}\n".format(intWrote, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone.fileno()) + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access + db = self._db + dbc = wt.getDbConn() + numTables = self.LARGE_NUMBER_OF_TABLES if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + tblSeq = list(range(numTables )) + random.shuffle(tblSeq) # now we have random sequence + for i in tblSeq: + if (i in self.activeTable): # wow already active + # print("x", end="", flush=True) # concurrent insertion + Progress.emit(Progress.CONCURRENT_INSERTION) + else: + self.activeTable.add(i) # marking it active + + dbName = db.getName() + sTable = db.getFixedSuperTable() + regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) + fullTableName = dbName + '.' + regTableName + # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" + sTable.ensureRegTable(self, wt.getDbConn(), regTableName) # Ensure the table exists + # self._unlockTable(fullTableName) + + self._deleteData(db, dbc, regTableName, te) + + self.activeTable.discard(i) # not raising an error, unlike remove + class ThreadStacks: # stack info for all threads def __init__(self): @@ -2259,7 +2690,8 @@ class ThreadStacks: # stack info for all threads # Now print print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(shortTid)) lastSqlForThread = DbConn.fetchSqlForThread(shortTid) - print("Last SQL statement attempted from thread {} is: {}".format(shortTid, lastSqlForThread)) + time_cost = DbConn.get_time_cost() + print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, time_cost ,lastSqlForThread)) stackFrame = 0 for frame in stack: # was using: reversed(stack) # print(frame) From 8a5427c95bd27eb55397d8a17ff76892a1b47a5e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 8 Nov 2022 09:41:42 +0800 Subject: [PATCH 003/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 39 ++++++++++------ tests/pytest/crash_gen/shared/db.py | 57 +++++++++++++++++++++--- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index d8946780a2..1ef89dec4e 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1761,7 +1761,10 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - + + if sTable.hasStreams(wt.getDbConn()) or sTable.hasStreamTables(wt.getDbConn()): + sTable.dropStreams(wt.getDbConn()) + sTable.dropStreamTables(wt.getDbConn()) sTable.create(wt.getDbConn(), {'ts': TdDataType.TIMESTAMP, 'speed': TdDataType.INT, 'color': TdDataType.BINARY16}, { 'b': TdDataType.BINARY200, 'f': TdDataType.FLOAT}, @@ -2557,16 +2560,20 @@ class TaskDeleteData(StateTransitionTask): # Now read it back and verify, we might encounter an error if table is dropped if Config.getConfig().verify_data: # only if command line asks for it try: - readBack = dbc.queryScalar("SELECT * from {}.{} WHERE ts='{}'". + dbc.query("SELECT * from {}.{} WHERE ts='{}'". format(db.getName(), regTableName, nextTick)) - if readBack == None : - pass + result = dbc.getQueryResult() + if len(result)==0: + # means data has been delete + print("D1",end="") # DF means delete failed + else: + print("DF",end="") # DF means delete failed except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result - print("D1",end="") # D1 means delete data success and only 1 record + # if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + # print("D1",end="") # D1 means delete data success and only 1 record - elif errno in [0x218, 0x362]: # table doesn't exist + if errno in [0x218, 0x362,0x2662]: # table doesn't exist # do nothing pass else: @@ -2612,16 +2619,20 @@ class TaskDeleteData(StateTransitionTask): # Now read it back and verify, we might encounter an error if table is dropped if Config.getConfig().verify_data: # only if command line asks for it try: - readBack = dbc.queryScalar("SELECT * from {}.{} ". - format(db.getName(), regTableName)) - if readBack == None : - pass + dbc.query("SELECT * from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + result = dbc.getQueryResult() + if len(result)==0: + # means data has been delete + print("DA",end="") + else: + print("DF",end="") # DF means delete failed except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result - print("Da",end="") # Da means delete data success and for all datas + # if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + # print("Da",end="") # Da means delete data success and for all datas - elif errno in [0x218, 0x362]: # table doesn't exist + if errno in [0x218, 0x362,0x2662]: # table doesn't exist # do nothing pass else: diff --git a/tests/pytest/crash_gen/shared/db.py b/tests/pytest/crash_gen/shared/db.py index 60c830f4f7..f97e4025e5 100644 --- a/tests/pytest/crash_gen/shared/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -26,9 +26,12 @@ class DbConn: TYPE_NATIVE = "native-c" TYPE_REST = "rest-api" TYPE_INVALID = "invalid" + + # class variables lastSqlFromThreads : dict[int, str] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 + spendThreads : dict[int, float] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 @classmethod def saveSqlForCurrentThread(cls, sql: str): @@ -37,15 +40,36 @@ class DbConn: run into a dead-lock situation, we can pick out the deadlocked thread, and use that information to find what what SQL statement is stuck. ''' + th = threading.current_thread() shortTid = th.native_id % 10000 #type: ignore cls.lastSqlFromThreads[shortTid] = sql # Save this for later @classmethod - def fetchSqlForThread(cls, shortTid : int) -> str : + def fetchSqlForThread(cls, shortTid : int) -> str : + + print("=======================") if shortTid not in cls.lastSqlFromThreads: raise CrashGenError("No last-attempted-SQL found for thread id: {}".format(shortTid)) - return cls.lastSqlFromThreads[shortTid] + return cls.lastSqlFromThreads[shortTid] + + + @classmethod + def sql_exec_spend(cls, cost: float): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + cls.spendThreads[shortTid] = cost # Save this for later + + @classmethod + def get_time_cost(cls) ->float: + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + return cls.spendThreads.get(shortTid) @classmethod def create(cls, connType, dbTarget): @@ -61,6 +85,7 @@ class DbConn: def createNative(cls, dbTarget) -> DbConn: return cls.create(cls.TYPE_NATIVE, dbTarget) + @classmethod def createRest(cls, dbTarget) -> DbConn: return cls.create(cls.TYPE_REST, dbTarget) @@ -75,6 +100,7 @@ class DbConn: return "[DbConn: type={}, target={}]".format(self._type, self._dbTarget) def getLastSql(self): + return self._lastSql def open(self): @@ -184,13 +210,19 @@ class DbConnRest(DbConn): def _doSql(self, sql): self._lastSql = sql # remember this, last SQL attempted self.saveSqlForCurrentThread(sql) # Save in global structure too. #TODO: combine with above - try: + time_cost = -1 + time_start = time.time() + try: r = requests.post(self._url, data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) + auth = HTTPBasicAuth('root', 'taosdata')) except: print("REST API Failure (TODO: more info here)") + self.sql_exec_spend(-2) raise + finally: + time_cost = time.time()- time_start + self.sql_exec_spend(time_cost) rj = r.json() # Sanity check for the "Json Result" if ('status' not in rj): @@ -223,6 +255,8 @@ class DbConnRest(DbConn): "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) return nRows + + def query(self, sql): # return rows affected return self.execute(sql) @@ -336,6 +370,7 @@ class MyTDSql: raise return self.affectedRows + class DbTarget: def __init__(self, cfgPath, hostAddr, port): self.cfgPath = cfgPath @@ -355,6 +390,7 @@ class DbConnNative(DbConn): # _connInfoDisplayed = False # TODO: find another way to display this totalConnections = 0 # Not private totalRequests = 0 + time_cost = -1 def __init__(self, dbTarget): super().__init__(dbTarget) @@ -413,8 +449,19 @@ class DbConnNative(DbConn): "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql + time_cost = -1 + nRows = 0 + time_start = time.time() self.saveSqlForCurrentThread(sql) # Save in global structure too. #TODO: combine with above - nRows = self._tdSql.execute(sql) + try: + nRows= self._tdSql.execute(sql) + except Exception as e: + self.sql_exec_spend(-2) + finally: + time_cost = time.time() - time_start + self.sql_exec_spend(time_cost) + + cls = self.__class__ cls.totalRequests += 1 Logging.debug( From 7aac1066507d69f55ed3317cfcd6228d2673ea3f Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 8 Nov 2022 10:54:57 +0800 Subject: [PATCH 004/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 149 +++++++++++++---------- 1 file changed, 87 insertions(+), 62 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 1ef89dec4e..8d2fb65ca8 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1712,30 +1712,66 @@ class TaskCreateStream(StateTransitionTask): ''' stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) + aggExpr = Dice.choice([ + 'count(*)', + 'avg(speed)', + # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable + 'sum(speed)', + 'stddev(speed)', + # SELECTOR functions + 'min(speed)', + 'max(speed)', + 'first(speed)', + 'last(speed)', + 'top(speed, 50)', # TODO: not supported? + 'bottom(speed, 50)', # TODO: not supported? + 'apercentile(speed, 10)', # TODO: TD-1316 + 'last_row(*)', # TODO: commented out per TD-3231, we should re-create + # Transformation Functions + # 'diff(speed)', # TODO: no supported?! + 'spread(speed)', + 'elapsed(ts)', + 'mode(speed)', + 'bottom(speed,1)', + 'top(speed,1)', + 'tail(speed,1)', + 'unique(color)', + 'csum(speed)', + 'DERIVATIVE(speed,1s,1)', + 'diff(speed,1)', + 'irate(speed)', + 'mavg(speed,3)', + 'sample(speed,5)', + 'STATECOUNT(speed,"LT",1)', + 'STATEDURATION(speed,"LT",1)', + 'twa(speed)' + + ]) # TODO: add more from 'top' + if sub_tables: if sub_tables: # if not empty sub_tbname = sub_tables[0] - # create stream with query above sub_table - stream_sql = 'create stream {} into {}.{} as select count(*), avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,dbname,sub_tbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) try: self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [0x03f0]: # stream already exists # stream need drop before drop table pass - sTable.setStreamName(sub_stream_name) else: pass else: - stream_sql = 'create stream {} into {}.{} as select count(*), avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,dbname,stbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) try: self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [0x03f0]: # stream already exists @@ -1779,16 +1815,10 @@ class TdSuperTable: def __init__(self, stName, dbName): self._stName = stName self._dbName = dbName - self._streamName = [] def getName(self): return self._stName - def setStreamName(self,name): - self._streamName.append(name) - - def getStreamName(self): - return self._streamName def drop(self, dbc, skipCheck = False): dbName = self._dbName @@ -2000,38 +2030,38 @@ class TdSuperTable: if not doAggr: # don't do aggregate query, just simple one commonExpr = Dice.choice([ '*', - # 'abs(speed)', - # 'acos(speed)', - # 'asin(speed)', - # 'atan(speed)', - # 'ceil(speed)', - # 'cos(speed)', - # 'cos(speed)', - # 'floor(speed)', - # 'log(speed,2)', - # 'pow(speed,2)', - # 'round(speed)', - # 'sin(speed)', - # 'sqrt(speed)', - # 'char_length(color)', - # 'concat(color,color)', - # 'concat_ws(" ", color,color," ")', - # 'length(color)', - # 'lower(color)', - # 'ltrim(color)', - # 'substr(color , 2)', - # 'upper(color)', - # 'cast(speed as double)', - # 'cast(ts as bigint)', - # # 'TO_ISO8601(color)', - # # 'TO_UNIXTIMESTAMP(ts)', - # 'now()', - # 'timediff(ts,now)', - # 'timezone()', - # 'TIMETRUNCATE(ts,1s)', - # 'TIMEZONE()', - # 'TODAY()', - # 'distinct(color)' + 'abs(speed)', + 'acos(speed)', + 'asin(speed)', + 'atan(speed)', + 'ceil(speed)', + 'cos(speed)', + 'cos(speed)', + 'floor(speed)', + 'log(speed,2)', + 'pow(speed,2)', + 'round(speed)', + 'sin(speed)', + 'sqrt(speed)', + 'char_length(color)', + 'concat(color,color)', + 'concat_ws(" ", color,color," ")', + 'length(color)', + 'lower(color)', + 'ltrim(color)', + 'substr(color , 2)', + 'upper(color)', + 'cast(speed as double)', + 'cast(ts as bigint)', + # 'TO_ISO8601(color)', + # 'TO_UNIXTIMESTAMP(ts)', + 'now()', + 'timediff(ts,now)', + 'timezone()', + 'TIMETRUNCATE(ts,1s)', + 'TIMEZONE()', + 'TODAY()', + 'distinct(color)' ] ) ret.append(SqlQuery( # reg table @@ -2057,21 +2087,21 @@ class TdSuperTable: # Transformation Functions # 'diff(speed)', # TODO: no supported?! 'spread(speed)', - # 'elapsed(ts)', - # 'mode(speed)', - # 'bottom(speed,1)', - # 'top(speed,1)', - # 'tail(speed,1)', - # 'unique(color)', - # 'csum(speed)', - # 'DERIVATIVE(speed,1s,1)', - # 'diff(speed,1)', - # 'irate(speed)', - # 'mavg(speed,3)', - # 'sample(speed,5)', - # 'STATECOUNT(speed,"LT",1)', - # 'STATEDURATION(speed,"LT",1)', - # 'twa(speed)' + 'elapsed(ts)', + 'mode(speed)', + 'bottom(speed,1)', + 'top(speed,1)', + 'tail(speed,1)', + 'unique(color)', + 'csum(speed)', + 'DERIVATIVE(speed,1s,1)', + 'diff(speed,1)', + 'irate(speed)', + 'mavg(speed,3)', + 'sample(speed,5)', + 'STATECOUNT(speed,"LT",1)', + 'STATEDURATION(speed,"LT",1)', + 'twa(speed)' @@ -2181,7 +2211,6 @@ class TaskDropSuperTable(StateTransitionTask): isSuccess = True for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - streams_prix = self._db.getName() try: self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL @@ -2189,10 +2218,6 @@ class TaskDropSuperTable(StateTransitionTask): pass - - - - if (not tickOutput): tickOutput = True # Print only one time if isSuccess: From 937631085f2956529e501dcf09eb83d47f984378 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 8 Nov 2022 11:44:42 +0800 Subject: [PATCH 005/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 8d2fb65ca8..1701c293a6 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1712,7 +1712,7 @@ class TaskCreateStream(StateTransitionTask): ''' stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) - aggExpr = Dice.choice([ + aggExpr = Dice.choice([ CREATE STREAM avg_vol_s1 INTO avg_vol1 AS SELECT csum(current), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); 'count(*)', 'avg(speed)', # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable @@ -1723,27 +1723,9 @@ class TaskCreateStream(StateTransitionTask): 'max(speed)', 'first(speed)', 'last(speed)', - 'top(speed, 50)', # TODO: not supported? - 'bottom(speed, 50)', # TODO: not supported? 'apercentile(speed, 10)', # TODO: TD-1316 'last_row(*)', # TODO: commented out per TD-3231, we should re-create # Transformation Functions - # 'diff(speed)', # TODO: no supported?! - 'spread(speed)', - 'elapsed(ts)', - 'mode(speed)', - 'bottom(speed,1)', - 'top(speed,1)', - 'tail(speed,1)', - 'unique(color)', - 'csum(speed)', - 'DERIVATIVE(speed,1s,1)', - 'diff(speed,1)', - 'irate(speed)', - 'mavg(speed,3)', - 'sample(speed,5)', - 'STATECOUNT(speed,"LT",1)', - 'STATEDURATION(speed,"LT",1)', 'twa(speed)' ]) # TODO: add more from 'top' From 908f0d719f191feecf55a303e09c584689072a55 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 8 Nov 2022 11:47:00 +0800 Subject: [PATCH 006/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 1701c293a6..6c23158535 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1696,9 +1696,9 @@ class TaskCreateStream(StateTransitionTask): dbname = self._db.getName() sub_stream_name = dbname+ '_sub_stream' - sub_stream_tb_name = 'avg_sub' + sub_stream_tb_name = 'stream_tb_sub' super_stream_name = dbname+ '_super_stream' - super_stream_tb_name = 'avg_super' + super_stream_tb_name = 'stream_tb_super' if not self._db.exists(wt.getDbConn()): Logging.debug("Skipping task, no DB yet") return @@ -1892,7 +1892,7 @@ class TdSuperTable: def hasStreamTables(self,dbc: DbConn): - return dbc.query("show {}.stables like 'avg%'".format(self._dbName)) > 0 + return dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) > 0 def hasStreams(self,dbc: DbConn): return dbc.query("show streams") > 0 @@ -1907,7 +1907,7 @@ class TdSuperTable: return not dbc.query("show streams ") > 0 def dropStreamTables(self, dbc: DbConn): - dbc.query("show {}.stables like 'avg%'".format(self._dbName)) + dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) StreamTables = dbc.getQueryResult() @@ -1915,7 +1915,7 @@ class TdSuperTable: if self.dropStreams: dbc.execute('drop table {}.{}'.format(self._dbName,StreamTable[0])) - return not dbc.query("show {}.stables like 'avg%'".format(self._dbName)) + return not dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) def ensureRegTable(self, task: Optional[Task], dbc: DbConn, regTableName: str): ''' From ca00459abb72a56a2b833ed225daf06532145510 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 8 Nov 2022 11:56:18 +0800 Subject: [PATCH 007/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 6c23158535..933d620824 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1712,7 +1712,7 @@ class TaskCreateStream(StateTransitionTask): ''' stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) - aggExpr = Dice.choice([ CREATE STREAM avg_vol_s1 INTO avg_vol1 AS SELECT csum(current), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + aggExpr = Dice.choice([ 'count(*)', 'avg(speed)', # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable @@ -1735,7 +1735,7 @@ class TaskCreateStream(StateTransitionTask): if sub_tables: # if not empty sub_tbname = sub_tables[0] # create stream with query above sub_table - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} where ts now -1h PARTITION BY tbname INTERVAL(5s) SLIDING(3s) FILL (prev) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) try: self.execWtSql(wt, stream_sql) Logging.debug("[OPS] stream is creating at {}".format(time.time())) @@ -1749,7 +1749,7 @@ class TaskCreateStream(StateTransitionTask): pass else: - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} where ts now -1h PARTITION BY tbname INTERVAL(5s) SLIDING(3s) FILL (prev) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) try: self.execWtSql(wt, stream_sql) From 59f50ae8c5ad38fc8e36eba5128ebdcda4f13f59 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:08:02 +0800 Subject: [PATCH 008/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 298 +++++++++++++++++++++-- 1 file changed, 271 insertions(+), 27 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 933d620824..6c52b459ca 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -37,6 +37,7 @@ import requests # from guppy import hpy import gc import taos +from taos.tmq import * from .shared.types import TdColumns, TdTags @@ -675,9 +676,10 @@ class AnyState: CAN_DROP_DB = 2 CAN_CREATE_FIXED_SUPER_TABLE = 3 CAN_CREATE_STREAM = 3 # super table must exists + CAN_CREATE_TOPIC = 3 # super table must exists + CAN_CREATE_CONSUMERS = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 CAN_ADD_DATA = 5 - CAN_DROP_STREAM = 5 CAN_READ_DATA = 6 CAN_DELETE_DATA = 6 @@ -730,6 +732,12 @@ class AnyState: return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] + def canCreateTopic(self): + return self._info[self.CAN_CREATE_TOPIC] + + def canCreateConsumers(self): + return self._info[self.CAN_CREATE_CONSUMERS] + def canCreateStream(self): return self._info[self.CAN_CREATE_STREAM] @@ -1679,9 +1687,22 @@ class TaskDropDb(StateTransitionTask): return state.canDropDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + + # drop topics before drop db + + if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) + self.execWtSql(wt, "drop database {}".format(self._db.getName())) + Logging.debug("[OPS] database dropped at {}".format(time.time())) + +''' +# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） + class TaskCreateStream(StateTransitionTask): @classmethod @@ -1707,9 +1728,9 @@ class TaskCreateStream(StateTransitionTask): # wt.execSql("use db") # should always be in place # create stream - ''' - CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); - ''' + + # CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) aggExpr = Dice.choice([ @@ -1735,7 +1756,7 @@ class TaskCreateStream(StateTransitionTask): if sub_tables: # if not empty sub_tbname = sub_tables[0] # create stream with query above sub_table - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} where ts now -1h PARTITION BY tbname INTERVAL(5s) SLIDING(3s) FILL (prev) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) try: self.execWtSql(wt, stream_sql) Logging.debug("[OPS] stream is creating at {}".format(time.time())) @@ -1749,7 +1770,7 @@ class TaskCreateStream(StateTransitionTask): pass else: - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} where ts now -1h PARTITION BY tbname INTERVAL(5s) SLIDING(3s) FILL (prev) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) try: self.execWtSql(wt, stream_sql) @@ -1759,8 +1780,142 @@ class TaskCreateStream(StateTransitionTask): if errno in [0x03f0]: # stream already exists # stream need drop before drop table pass +''' - +class TaskCreateTopic(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateTopic() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sub_topic_name = dbname+ '_sub_topic' + super_topic_name = dbname+ '_super_topic' + stable_topic = dbname+ '_stable_topic' + db_topic = 'database_' + dbname+ '_topics' + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create topic + + # create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1; + + stbname =sTable.getName() + sub_tables = sTable.getRegTables(wt.getDbConn()) + scalarExpr = Dice.choice([ '*','speed','color', + 'abs(speed)', + 'acos(speed)', + 'asin(speed)', + 'atan(speed)', + 'ceil(speed)', + 'cos(speed)', + 'cos(speed)', + 'floor(speed)', + 'log(speed,2)', + 'pow(speed,2)', + 'round(speed)', + 'sin(speed)', + 'sqrt(speed)', + 'char_length(color)', + 'concat(color,color)', + 'concat_ws(" ", color,color," ")', + 'length(color)', + 'lower(color)', + 'ltrim(color)', + 'substr(color , 2)', + 'upper(color)', + 'cast(speed as double)', + 'cast(ts as bigint)', + + ]) # TODO: add more from 'top' + if Dice.throw(3)==0: + + if sub_tables: + + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create stream with query above sub_table + topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + + else: + pass + + else: + topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] subquery topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + elif Dice.throw(3)==1: + topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + elif Dice.throw(3)==2: + topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] db topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + else: + pass + + +class TaskCreateConsumers(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateConsumers() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create Consumers + if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) class TaskCreateSuperTable(StateTransitionTask): @@ -1780,9 +1935,6 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - if sTable.hasStreams(wt.getDbConn()) or sTable.hasStreamTables(wt.getDbConn()): - sTable.dropStreams(wt.getDbConn()) - sTable.dropStreamTables(wt.getDbConn()) sTable.create(wt.getDbConn(), {'ts': TdDataType.TIMESTAMP, 'speed': TdDataType.INT, 'color': TdDataType.BINARY16}, { 'b': TdDataType.BINARY200, 'f': TdDataType.FLOAT}, @@ -1797,6 +1949,8 @@ class TdSuperTable: def __init__(self, stName, dbName): self._stName = stName self._dbName = dbName + self._consumerLists = {} + self._ConsumerInsts = [] def getName(self): return self._stName @@ -1806,6 +1960,12 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName + if self.hasStreams(dbc): + self.dropStreams(dbc) + self.dropStreamTables(dbc) + if self.hasTopics(dbc): + self.dropTopics(dbName,None) + self.dropTopics(dbName,self._stName) try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: @@ -1843,12 +2003,19 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: + if self.hasStreams(dbc): + self.dropStreams(dbc) + self.dropStreamTables(dbc) + + # drop topics before drop stables + if self.hasTopics(dbc): + self.dropTopics(dbc,self._dbName,None) + self.dropTopics(dbc,self._dbName,self._stName ) + dbc.execute("DROP TABLE {}".format(fullTableName)) - pass - - + pass # dbc.execute("DROP TABLE {}".format(fullTableName)) else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -1863,7 +2030,47 @@ class TdSuperTable: ) else: sql += " TAGS (dummy int) " - dbc.execute(sql) + dbc.execute(sql) + + def createConsumer(self, dbc: DbConn , Consumer_nums): + + def generateConsumer(current_topic_list): + conf = TaosTmqConf() + conf.set("group.id", "tg2") + conf.set("td.connect.user", "root") + conf.set("td.connect.pass", "taosdata") + conf.set("enable.auto.commit", "true") + def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + conf.set_auto_commit_cb(tmq_commit_cb_print, None) + consumer = conf.new_consumer() + topic_list = TaosTmqList() + for topic in current_topic_list: + topic_list.append(topic) + consumer.subscribe(topic_list) + time.sleep(5) # consumer work only 5 sec ,and then it will exit + try: + consumer.unsubscribe() + except Exception as e : + pass + return + + # mulit Consumer + current_topic_list = self.getTopicLists(dbc) + for i in range(Consumer_nums): + consumer_inst = threading.Thread(target=generateConsumer, args=(current_topic_list,)) + self._ConsumerInsts.append(consumer_inst) + + for ConsumerInst in self._ConsumerInsts: + ConsumerInst.start() + for ConsumerInst in self._ConsumerInsts: + ConsumerInst.join() + + def getTopicLists(self, dbc: DbConn): + dbc.query("show topics ") + topics = dbc.getQueryResult() + topicLists = [v[0] for v in topics] + return topicLists def getRegTables(self, dbc: DbConn): dbName = self._dbName @@ -1878,12 +2085,7 @@ class TdSuperTable: return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation def hasRegTables(self, dbc: DbConn): - # print(self._stName) - # dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) - # print(dbc.getQueryResult()) - if self.hasStreamTables(dbc) or self.hasStreams(dbc): - if self.dropStreams(dbc): - self.dropStreamTables(dbc) + if dbc.existsSuperTable(self._stName): return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 @@ -1896,7 +2098,41 @@ class TdSuperTable: def hasStreams(self,dbc: DbConn): return dbc.query("show streams") > 0 - + + def hasTopics(self,dbc: DbConn): + + return dbc.query("show topics") > 0 + + def dropTopics(self,dbc: DbConn , dbname=None,stb_name=None): + dbc.query("show topics ") + topics = dbc.getQueryResult() + + if dbname !=None and stb_name == None : + + for topic in topics: + if dbname in topic[0] and topic[0].startswith("database"): + try: + dbc.execute('drop topic {}'.format(topic[0])) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03EB]: # Topic subscribed cannot be dropped + pass + # for subsript in subscriptions: + + else: + pass + + pass + return True + elif dbname !=None and stb_name!= None: + for topic in topics: + if topic[0].startswith(self._dbName) and topic[0].endswith('topic'): + dbc.execute('drop topic {}'.format(topic[0])) + return True + else: + return True + pass + def dropStreams(self,dbc:DbConn): dbc.query("show streams ") Streams = dbc.getQueryResult() @@ -1912,7 +2148,7 @@ class TdSuperTable: StreamTables = dbc.getQueryResult() for StreamTable in StreamTables: - if self.dropStreams: + if self.dropStreams(dbc): dbc.execute('drop table {}.{}'.format(self._dbName,StreamTable[0])) return not dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) @@ -2083,11 +2319,7 @@ class TdSuperTable: 'sample(speed,5)', 'STATECOUNT(speed,"LT",1)', 'STATEDURATION(speed,"LT",1)', - 'twa(speed)' - - - - + 'twa(speed)' ]) # TODO: add more from 'top' @@ -2194,6 +2426,7 @@ class TaskDropSuperTable(StateTransitionTask): for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) try: + self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: @@ -2209,6 +2442,17 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() + + # drop streams before drop stables + if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): + self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) + self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + + # drop topics before drop stables + if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) + try: self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) except taos.error.ProgrammingError as err: From dde87a374601d6511c4ca808dd5696dc8c976ba5 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:21:43 +0800 Subject: [PATCH 009/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 6c52b459ca..dbaa8b8ef2 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1372,6 +1372,7 @@ class Task(): 0x03D3, # Conflict transaction not completed 0x0707, # Query not ready , it always occur at replica 3 0x707, # Query not ready + 0x396, # Database in creating status From 05f0ac62e7c5d6a824c24a258841b1c9fadf7254 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:32:31 +0800 Subject: [PATCH 010/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index dbaa8b8ef2..eeffe15a88 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1373,7 +1373,7 @@ class Task(): 0x0707, # Query not ready , it always occur at replica 3 0x707, # Query not ready 0x396, # Database in creating status - + 0x386, # Database in droping status 1000 # REST catch-all error @@ -2052,7 +2052,7 @@ class TdSuperTable: time.sleep(5) # consumer work only 5 sec ,and then it will exit try: consumer.unsubscribe() - except Exception as e : + except TmqError as e : pass return From 3e133a8661ab4b15fc09482b054787f3079279ab Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 09:35:22 +0800 Subject: [PATCH 011/165] updaate --- tests/pytest/crash_gen/crash_gen_main.py | 32 +++++++++++++++--------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index eeffe15a88..18c3957630 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -255,7 +255,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 1200 # Normal: 120 + WORKER_THREAD_TIMEOUT = 120 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -1374,6 +1374,7 @@ class Task(): 0x707, # Query not ready 0x396, # Database in creating status 0x386, # Database in droping status + 0x03E1, # failed on tmq_subscribe ,topic not exist 1000 # REST catch-all error @@ -1905,18 +1906,23 @@ class TaskCreateConsumers(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - return state.canCreateConsumers() + return state.canCreateConsumers() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - dbname = self._db.getName() - - sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place - # create Consumers - if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task - if sTable.hasTopics(wt.getDbConn()): - sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + if Config.getConfig().connector_type == 'native': + dbname = self._db.getName() + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create Consumers + if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + else: + print(" restful not support tmq consumers") + return class TaskCreateSuperTable(StateTransitionTask): @@ -2048,7 +2054,10 @@ class TdSuperTable: topic_list = TaosTmqList() for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) + try: + consumer.subscribe(topic_list) + except TmqError as e : + pass time.sleep(5) # consumer work only 5 sec ,and then it will exit try: consumer.unsubscribe() @@ -3326,4 +3335,3 @@ class Container(): return self._verifyValidProperty(name) self._cargo[name] = value - From 86b5b89531772a795f179c851f16b7ff5f282366 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 10:37:16 +0800 Subject: [PATCH 012/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 18c3957630..7d5d1c0c8a 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1367,6 +1367,7 @@ class Task(): 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched 0x2662, # Table does not exist #TODO: what about 2603 above? + 0x2600, # database not specified, SQL: show stables , database droped , and show tables 0x032C, # Object is creating 0x032D, # Object is dropping 0x03D3, # Conflict transaction not completed From bf30350195ee6d91ece72d9b29057358351a9229 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Thu, 10 Nov 2022 18:47:21 +0800 Subject: [PATCH 013/165] test:run cast function in query mode 4 --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6775760ee4..dda053ccf8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -968,6 +968,7 @@ ,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 4 From 20071f09a729da0280c0ca82f76b410463f5e30a Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 20:23:00 +0800 Subject: [PATCH 014/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 218 ++++++++++++++++++----- tests/pytest/crash_gen/shared/db.py | 24 ++- 2 files changed, 196 insertions(+), 46 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 7d5d1c0c8a..ba501c3e78 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -420,10 +420,12 @@ class ThreadCoordinator: except threading.BrokenBarrierError as err: self._execStats.registerFailure("Aborted due to worker thread timeout") Logging.error("\n") + Logging.error("Main loop aborted, caused by worker thread(s) time-out of {} seconds".format( ThreadCoordinator.WORKER_THREAD_TIMEOUT)) Logging.error("TAOS related threads blocked at (stack frames top-to-bottom):") ts = ThreadStacks() + ts.record_current_time(time.time()) # record thread exit time at current moment ts.print(filterInternal=True) workerTimeout = True @@ -547,7 +549,12 @@ class ThreadCoordinator: # pick a task type for current state db = self.pickDatabase() - taskType = db.getStateMachine().pickTaskType() # dynamic name of class + if Dice.throw(2)==1: + taskType = db.getStateMachine().pickTaskType() # dynamic name of class + else: + taskType = db.getStateMachine().balance_pickTaskType() # and an method can get balance task types + pass + return taskType(self._execStats, db) # create a task from it def resetExecutedTasks(self): @@ -679,6 +686,8 @@ class AnyState: CAN_CREATE_TOPIC = 3 # super table must exists CAN_CREATE_CONSUMERS = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 + CAN_DROP_TOPIC = 4 + CAN_DROP_STREAM = 4 CAN_ADD_DATA = 5 CAN_READ_DATA = 6 CAN_DELETE_DATA = 6 @@ -734,13 +743,19 @@ class AnyState: def canCreateTopic(self): return self._info[self.CAN_CREATE_TOPIC] + + def canDropTopic(self): + return self._info[self.CAN_DROP_TOPIC] def canCreateConsumers(self): return self._info[self.CAN_CREATE_CONSUMERS] - def canCreateStream(self): + def canCreateStreams(self): return self._info[self.CAN_CREATE_STREAM] + def canDropStream(self): + return self._info[self.CAN_DROP_STREAM] + def canAddData(self): return self._info[self.CAN_ADD_DATA] @@ -919,7 +934,7 @@ class StateHasData(AnyState): ): # only if we didn't create one # we shouldn't have dropped it self.assertNoTask(tasks, TaskDropDb) - if (not self.hasTask(tasks, TaskCreateSuperTable) + if not( self.hasTask(tasks, TaskCreateSuperTable) ): # if we didn't create the table # we should not have a task that drops it self.assertNoTask(tasks, TaskDropSuperTable) @@ -1075,6 +1090,28 @@ class StateMechine: # Logging.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) return taskTypes[i] + def balance_pickTaskType(self): + # all the task types we can choose from at curent state + BasicTypes = self.getTaskTypes() + weightsTypes = BasicTypes.copy() + + # this matrixs can balance the Frequency of different types of tasks + weight_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , + 'TaskReadData':50 , 'TaskDropSuperTable':5 , 'TaskAlterTags':3 , 'TaskAddData':10, + 'TaskDeleteData':10 , 'TaskCreateDb':10 , 'TaskCreateStream': 3, 'TaskCreateTopic' :3, + 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # task type : weghts matrixs + + for task , weights in weight_matrixs.items(): + + for basicType in BasicTypes: + if basicType.__name__ == task: + for _ in range(weights): + weightsTypes.append(basicType) + + task = random.sample(weightsTypes,1) + return task[0] + + # ref: # https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ def _weighted_choice_sub(self, weights) -> int: @@ -1376,6 +1413,10 @@ class Task(): 0x396, # Database in creating status 0x386, # Database in droping status 0x03E1, # failed on tmq_subscribe ,topic not exist + 0x03ed , # Topic must be dropped first, SQL: drop database db_0 + 0x0203 , # Invalid value + + 1000 # REST catch-all error @@ -1693,19 +1734,24 @@ class TaskDropDb(StateTransitionTask): # drop topics before drop db - if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) - - self.execWtSql(wt, "drop database {}".format(self._db.getName())) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) + try: + self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x0203]: # drop maybe failed + pass Logging.debug("[OPS] database dropped at {}".format(time.time())) -''' + # Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） + class TaskCreateStream(StateTransitionTask): @classmethod @@ -1714,7 +1760,7 @@ class TaskCreateStream(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - return state.canCreateStream() + return state.canCreateStreams() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): dbname = self._db.getName() @@ -1783,7 +1829,6 @@ class TaskCreateStream(StateTransitionTask): if errno in [0x03f0]: # stream already exists # stream need drop before drop table pass -''' class TaskCreateTopic(StateTransitionTask): @@ -1855,7 +1900,7 @@ class TaskCreateTopic(StateTransitionTask): Logging.debug("[OPS] topic is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists + if errno in [0x03f0 ]: # topic already exists # topic need drop before drop table pass @@ -1877,7 +1922,7 @@ class TaskCreateTopic(StateTransitionTask): topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) try: self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) + self.queryWtSql(wt, topic_sql) Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) @@ -1897,7 +1942,81 @@ class TaskCreateTopic(StateTransitionTask): pass else: pass - + +class TaskDropTopics(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropTopic() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + tblName = sTable.getName() + if sTable.hasTopics(wt.getDbConn()): + sTable.dropTopics(wt.getDbConn(),dbname,None) # drop topics of database + sTable.dropTopics(wt.getDbConn(),dbname,tblName) # drop topics of stable + +class TaskDropStreams(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreams(wt.getDbConn()): + sTable.dropStreams(wt.getDbConn()) # drop stream of database + # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable + +class TaskDropStreamTables(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreamTables(wt.getDbConn()): + # sTable.dropStreams(wt.getDbConn()) + sTable.dropStreamTables(wt.getDbConn()) # drop stream tables class TaskCreateConsumers(StateTransitionTask): @@ -1918,9 +2037,10 @@ class TaskCreateConsumers(StateTransitionTask): # wt.execSql("use db") # should always be in place # create Consumers - if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task - if sTable.hasTopics(wt.getDbConn()): - sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + # if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + pass else: print(" restful not support tmq consumers") return @@ -1968,17 +2088,17 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName - if self.hasStreams(dbc): - self.dropStreams(dbc) - self.dropStreamTables(dbc) - if self.hasTopics(dbc): - self.dropTopics(dbName,None) - self.dropTopics(dbName,self._stName) + # if self.hasStreams(dbc): + # self.dropStreams(dbc) + # self.dropStreamTables(dbc) + # if self.hasTopics(dbc): + # self.dropTopics(dbName,None) + # self.dropTopics(dbName,self._stName) try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table pass # # stream need drop before drop table # for stream in self.getStreamName(): @@ -2011,17 +2131,21 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: - if self.hasStreams(dbc): - self.dropStreams(dbc) - self.dropStreamTables(dbc) + # if self.hasStreams(dbc): + # self.dropStreams(dbc) + # self.dropStreamTables(dbc) - # drop topics before drop stables - if self.hasTopics(dbc): - self.dropTopics(dbc,self._dbName,None) - self.dropTopics(dbc,self._dbName,self._stName ) + # # drop topics before drop stables + # if self.hasTopics(dbc): + # self.dropTopics(dbc,self._dbName,None) + # self.dropTopics(dbc,self._dbName,self._stName ) - - dbc.execute("DROP TABLE {}".format(fullTableName)) + try: + dbc.execute("DROP TABLE {}".format(fullTableName)) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table + pass pass # dbc.execute("DROP TABLE {}".format(fullTableName)) @@ -2124,6 +2248,7 @@ class TdSuperTable: if dbname in topic[0] and topic[0].startswith("database"): try: dbc.execute('drop topic {}'.format(topic[0])) + Logging.debug("[OPS] topic {} is droping at {}".format(topic,time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [0x03EB]: # Topic subscribed cannot be dropped @@ -2139,6 +2264,7 @@ class TdSuperTable: for topic in topics: if topic[0].startswith(self._dbName) and topic[0].endswith('topic'): dbc.execute('drop topic {}'.format(topic[0])) + Logging.debug("[OPS] topic {} is droping at {}".format(topic,time.time())) return True else: return True @@ -2454,15 +2580,16 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - # drop streams before drop stables - if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): - self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) - self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + # # drop streams before drop stables + # if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): + # self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) + # self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + + # # drop topics before drop stables + # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) - # drop topics before drop stables - if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) try: self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) @@ -2948,6 +3075,9 @@ class ThreadStacks: # stack info for all threads shortTid = th.native_id % 10000 #type: ignore self._allStacks[shortTid] = stack # Was using th.native_id + def record_current_time(self,current_time): + self.current_time = current_time + def print(self, filteredEndName = None, filterInternal = False): for shortTid, stack in self._allStacks.items(): # for each thread, stack frames top to bottom lastFrame = stack[-1] @@ -2962,9 +3092,11 @@ class ThreadStacks: # stack info for all threads continue # ignore # Now print print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(shortTid)) + lastSqlForThread = DbConn.fetchSqlForThread(shortTid) - time_cost = DbConn.get_time_cost() - print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, time_cost ,lastSqlForThread)) + last_sql_commit_time = DbConn.get_save_sql_time(shortTid) + # time_cost = DbConn.get_time_cost() + print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, self.current_time-last_sql_commit_time ,lastSqlForThread)) stackFrame = 0 for frame in stack: # was using: reversed(stack) # print(frame) diff --git a/tests/pytest/crash_gen/shared/db.py b/tests/pytest/crash_gen/shared/db.py index f97e4025e5..05711efbc6 100644 --- a/tests/pytest/crash_gen/shared/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -32,7 +32,7 @@ class DbConn: # class variables lastSqlFromThreads : dict[int, str] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 spendThreads : dict[int, float] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 - + current_time : dict[int, float] = {} # save current time @classmethod def saveSqlForCurrentThread(cls, sql: str): ''' @@ -44,6 +44,7 @@ class DbConn: th = threading.current_thread() shortTid = th.native_id % 10000 #type: ignore cls.lastSqlFromThreads[shortTid] = sql # Save this for later + cls.record_save_sql_time() @classmethod def fetchSqlForThread(cls, shortTid : int) -> str : @@ -53,6 +54,25 @@ class DbConn: raise CrashGenError("No last-attempted-SQL found for thread id: {}".format(shortTid)) return cls.lastSqlFromThreads[shortTid] + @classmethod + def get_save_sql_time(cls, shortTid : int): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + return cls.current_time[shortTid] + + @classmethod + def record_save_sql_time(cls): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + cls.current_time[shortTid] = float(time.time()) # Save this for later @classmethod def sql_exec_spend(cls, cost: float): @@ -460,7 +480,6 @@ class DbConnNative(DbConn): finally: time_cost = time.time() - time_start self.sql_exec_spend(time_cost) - cls = self.__class__ cls.totalRequests += 1 @@ -541,4 +560,3 @@ class DbManager(): self._dbConn.close() self._dbConn = None Logging.debug("DbManager closed DB connection...") - From 4774696aaeddc6e7eda26a30729f3916922c84c8 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 20:32:08 +0800 Subject: [PATCH 015/165] update errorno --- tests/pytest/auto_crash_gen.py | 4 ++-- tests/pytest/auto_crash_gen_valgrind.py | 4 ++-- tests/pytest/auto_crash_gen_valgrind_cluster.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py index 02cca810a7..9c134e6d64 100755 --- a/tests/pytest/auto_crash_gen.py +++ b/tests/pytest/auto_crash_gen.py @@ -218,11 +218,11 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): if valgrind : - crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path ,arguments) + crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203 '%(crash_gen_path ,arguments) else: - crash_gen_cmd = 'cd %s && ./crash_gen.sh %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550'%(crash_gen_path ,arguments) + crash_gen_cmd = 'cd %s && ./crash_gen.sh %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203'%(crash_gen_path ,arguments) return crash_gen_cmd diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py index 1443dcd543..ce87fec684 100755 --- a/tests/pytest/auto_crash_gen_valgrind.py +++ b/tests/pytest/auto_crash_gen_valgrind.py @@ -219,11 +219,11 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): if valgrind : - crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path ,arguments) + crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203 '%(crash_gen_path ,arguments) else: - crash_gen_cmd = 'cd %s && ./crash_gen.sh %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550'%(crash_gen_path ,arguments) + crash_gen_cmd = 'cd %s && ./crash_gen.sh %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203'%(crash_gen_path ,arguments) return crash_gen_cmd diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py index 05cdaa6cc5..f4afa80afe 100755 --- a/tests/pytest/auto_crash_gen_valgrind_cluster.py +++ b/tests/pytest/auto_crash_gen_valgrind_cluster.py @@ -219,11 +219,11 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): if valgrind : - crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0707 '%(crash_gen_path ,arguments) + crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0707,0x0203 '%(crash_gen_path ,arguments) else: - crash_gen_cmd = 'cd %s && ./crash_gen.sh -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0014,0x0707'%(crash_gen_path ,arguments) + crash_gen_cmd = 'cd %s && ./crash_gen.sh -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0014,0x0707,0x0203'%(crash_gen_path ,arguments) return crash_gen_cmd From 4eb5c2cae7f146e190276472ec9b173e0b9996c8 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 11 Nov 2022 13:49:01 +0800 Subject: [PATCH 016/165] update --- tests/pytest/crash_gen/crash_gen_main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ba501c3e78..4fa19f543d 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -953,7 +953,7 @@ class StateMechine: except taos.error.ProgrammingError as err: Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) traceback.print_stack() - pass # re-throw + raise # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -1095,13 +1095,13 @@ class StateMechine: BasicTypes = self.getTaskTypes() weightsTypes = BasicTypes.copy() - # this matrixs can balance the Frequency of different types of tasks - weight_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , + # this matrixs can balance the Frequency of TaskTypes + balance_TaskType_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , 'TaskReadData':50 , 'TaskDropSuperTable':5 , 'TaskAlterTags':3 , 'TaskAddData':10, 'TaskDeleteData':10 , 'TaskCreateDb':10 , 'TaskCreateStream': 3, 'TaskCreateTopic' :3, - 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # task type : weghts matrixs + 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # TaskType : balance_matrixs of task - for task , weights in weight_matrixs.items(): + for task , weights in balance_TaskType_matrixs.items(): for basicType in BasicTypes: if basicType.__name__ == task: From 8c07f44f1374cf0cfaf87c43ae09fcade10276ba Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Tue, 15 Nov 2022 13:02:07 +0800 Subject: [PATCH 017/165] doc: updated contribution guidelines --- CONTRIBUTING-CN.md | 27 ++++++++------------- CONTRIBUTING.md | 60 +++++++++++++++++++++------------------------- 2 files changed, 37 insertions(+), 50 deletions(-) diff --git a/CONTRIBUTING-CN.md b/CONTRIBUTING-CN.md index 19f3000d45..efaa2077fe 100644 --- a/CONTRIBUTING-CN.md +++ b/CONTRIBUTING-CN.md @@ -7,25 +7,18 @@ - 任何用户都可以通过 **[GitHub issue tracker](https://github.com/taosdata/TDengine/issues)** 向我们报告错误。请您对所遇到的问题进行**详细描述**，最好提供重现错误的详细步骤。 - 欢迎提供包含由 Bug 生成的日志文件的附录。 -## 需要强调的代码提交规则 +## 代码提交规则 -- 在提交代码之前，需要**同意贡献者许可协议（CLA）**。点击 [TaosData CLA](https://cla-assistant.io/taosdata/TDengine) 阅读并签署协议。如果您不接受该协议，请停止提交。 -- 请在 [GitHub issue tracker](https://github.com/taosdata/TDengine/issues) 中解决问题或添加注册功能。 -- 如果在 [GitHub issue tracker](https://github.com/taosdata/TDengine/issues) 中没有找到相应的问题或功能，请**创建一个新的 issue**。 -- 将代码提交到我们的存储库时，请创建**包含问题编号的 PR**。 +1. 在提交代码之前，需要**同意贡献者许可协议（CLA）**。点击 [TaosData CLA](https://cla-assistant.io/taosdata/TDengine) 阅读并签署协议。如果您不接受该协议，请停止提交。 +2. 请在 [GitHub issue tracker](https://github.com/taosdata/TDengine/issues) 中解决问题或添加注册功能。 + 如果在 [GitHub issue tracker](https://github.com/taosdata/TDengine/issues) 中没有找到相应的问题或功能，请**创建一个新的 issue**。 + 将代码提交到我们的存储库时，请创建**包含问题编号的 PR**。 +3. 将TDengine仓库库fork到自己的账户中并创建分支(branch)。 + 注意：默认分支`main`不能直接接受PR，请基于开发分支`3.0`创建自己的分支。 + 注意：修改文档的分支要以`docs/`为开头，以免进行不必要的测试。 +4. 创建pull request，将自己的分支合并到开发分支`3.0`，我们开发团队将尽快审核。 -## 贡献指南 - -1. 请用友好的语气书写。 - -2. **主动语态**总体上优于被动语态。主动语态中的句子会突出执行动作的人，而不是被动语态突出动作的接受者。 - -3. 文档写作建议 - -- 正确拼写产品名称 “TDengine”。 “TD” 用大写字母，“TD” 和 “engine” 之间没有空格 **（正确拼写：TDengine）**。 -- 在句号或其他标点符号后只留一个空格。 - -4. 尽量**使用简单句**，而不是复杂句。 +如遇任何问题，请添加官方微信TDengineECO。我们的团队会帮忙解决。 ## 给贡献者的礼品 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5be84bec34..058c624e10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,40 +1,36 @@ -# Contributing +# Contributing to TDengine -We appreciate contributions from all developers. Feel free to follow us, fork the repository, report bugs, and even submit your code on GitHub. However, we would like developers to follow the guidelines in this document to ensure effective cooperation. +TDengine Community Edition is free, open-source software. Its development is led by the TDengine Team, but we welcome contributions from all community members and open-source developers. This document describes how you can contribute, no matter whether you're a user or a developer yourself. -## Reporting a bug +## Bug reports -- Any users can report bugs to us through the **[GitHub issue tracker](https://github.com/taosdata/TDengine/issues)**. We would appreciate if you could provide **a detailed description** of the problem you encountered, including steps to reproduce it. +All users can report bugs to us through the **[GitHub issue tracker](https://github.com/taosdata/TDengine/issues)**. To ensure that the development team can locate and resolve the issue that you experienced, please include the following in your bug report: -- Attaching log files caused by the bug is really appreciated. +- A detailed description of the issue, including the steps to reproduce it. +- Any log files that may be relevant to the issue. -## Guidelines for committing code +## Code contributions -- You must agree to the **Contributor License Agreement(CLA) before submitting your code patch**. Follow the **[TAOSData CLA](https://cla-assistant.io/taosdata/TDengine)** link to read through and sign the agreement. If you do not accept the agreement, your contributions cannot be accepted. +Developers are encouraged to submit patches to the project, and all contributions, from minor documentation changes to bug fixes, are appreciated by our team. To ensure that your code can be merged successfully and improve the experience for other community members, we ask that you go through the following procedure before submitting a pull request: -- Please solve an issue or add a feature registered in the **[GitHub issue tracker](https://github.com/taosdata/TDengine/issues)**. -- If no corresponding issue or feature is found in the issue tracker, please **create one**. -- When submitting your code to our repository, please create a pull request with the **issue number** included. +1. Read and accept the terms of the TAOS Data Contributor License Agreement (CLA) located at [https://cla-assistant.io/taosdata/TDengine](https://cla-assistant.io/taosdata/TDengine). -## Guidelines for communicating +2. For bug fixes, search the [GitHub issue tracker](https://github.com/taosdata/TDengine/issues) to check whether the bug has already been filed. + - If the bug that you want to fix already exists in the issue tracker, review the previous discussion before submitting your patch. + - If the bug that you want to fix does not exist in the issue tracker, click **New issue** and file a report. + - Ensure that you note the issue number in your pull request when you submit your patch. + +3. Fork our repository to your GitHub account and create a branch for your patch. + **Important:** The `main` branch is for stable versions and cannot accept patches directly. For all code and documentation changes, create your own branch from the development branch `3.0` and not from `main`. + Note: For a documentation change, ensure that the branch name starts with `docs/` so that the change can be merged without running tests. + +4. Create a pull request to merge your changes into the development branch `3.0`, and our team members will review the request as soon as possible. -1. Please be **nice and polite** in the description. -2. **Active voice is better than passive voice in general**. Sentences in the active voice will highlight who is performing the action rather than the recipient of the action highlighted by the passive voice. -3. Documentation writing advice +If you encounter any difficulties or problems in contributing your code, you can join our [Discord server](https://discord.com/invite/VZdSuUg4pS) and receive assistance from members of the TDengine Team. -- Spell the product name "TDengine" correctly. "TD" is written in capital letters, and there is no space between "TD" and "engine" (**Correct spelling: TDengine**). -- Please **capitalize the first letter** of every sentence. -- Leave **only one space** after periods or other punctuation marks. -- Use **American spelling**. -- When possible, **use second person** rather than first person (e.g.“You are recommended to use a reverse proxy such as Nginx.” rather than “We recommend to use a reverse proxy such as Nginx.”). +## Expressing our thanks -5. Use **simple sentences**, rather than complex sentences. - -## Gifts for the contributors - -Developers, as long as you contribute to TDengine, whether it's code contributions to fix bugs or feature requests, or documentation changes, **you are eligible for a very special Contributor Souvenir Gift!** - -**You can choose one of the following gifts:** +To thank community members for your support, we are offering a free gift to any developer who submits at least one contribution. You can choose one of the following items:

-The TDengine community is committed to making TDengine accepted and used by more developers. +If you would like to claim your gift, send an email to [developer@tdengine.com](mailto:developer@tdengine.com?subject=Claiming&20my%20developer%20gift) including the following information: -Just fill out the **Contributor Submission Form** to choose your desired gift. +- Your GitHub account name +- Your name and mailing address +- Your preferred gift -- [Contributor Submission Form](https://page.ma.scrmtech.com/form/index?pf_uid=27715_2095&id=12100) - -## Contact us - -If you have any problems or questions that need help from us, please feel free to add our WeChat account: TDengineECO. +Note: Limit one per person. \ No newline at end of file From aaef32ca6970a45546a617a25699c13a432156a1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Nov 2022 10:27:05 +0800 Subject: [PATCH 018/165] refact code for further code change --- include/common/tdataformat.h | 1 - source/dnode/vnode/src/inc/tsdb.h | 10 +++++----- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 2 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 6 +++--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2eda2f66cc..5d0b012fca 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -167,7 +167,6 @@ struct STSRow2 { #pragma pack(pop) struct STSRowBuilder { - // STSRow2 tsRow; int32_t szBuf; uint8_t *pBuf; }; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c6e2842c32..518cfb62db 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -56,7 +56,7 @@ typedef struct SDataFWriter SDataFWriter; typedef struct SDataFReader SDataFReader; typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; +typedef struct STSDBRowIter STSDBRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; @@ -111,9 +111,9 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); -// SRowIter -void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); -SColVal *tRowIterNext(SRowIter *pIter); +// STSDBRowIter +void tRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +SColVal *tRowIterNext(STSDBRowIter *pIter); // SRowMerger int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); @@ -562,7 +562,7 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; -struct SRowIter { +struct STSDBRowIter { TSDBROW *pRow; STSchema *pTSchema; SColVal colVal; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9f59707ddc..87506126c7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -595,7 +595,7 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; - SRowIter iter = {0}; + STSDBRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 0aa2c6ab83..3412e620c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -579,8 +579,8 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } -// SRowIter ====================================================== -void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +// STSDBRowIter ====================================================== +void tRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { pIter->pRow = pRow; if (pRow->type == 0) { ASSERT(pTSchema); @@ -594,7 +594,7 @@ void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { } } -SColVal *tRowIterNext(SRowIter *pIter) { +SColVal *tRowIterNext(STSDBRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); From 9c7913b00b1100cb48845af2cb640fd594a65f71 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Wed, 16 Nov 2022 10:39:37 +0800 Subject: [PATCH 019/165] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 244 +++++------------------ 1 file changed, 48 insertions(+), 196 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 4fa19f543d..90d27c3b8d 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1415,6 +1415,7 @@ class Task(): 0x03E1, # failed on tmq_subscribe ,topic not exist 0x03ed , # Topic must be dropped first, SQL: drop database db_0 0x0203 , # Invalid value + 0x03f0 , # Stream already exist , topic already exists @@ -1732,12 +1733,6 @@ class TaskDropDb(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # drop topics before drop db - - # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) try: self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists except taos.error.ProgrammingError as err: @@ -1748,10 +1743,6 @@ class TaskDropDb(StateTransitionTask): Logging.debug("[OPS] database dropped at {}".format(time.time())) - -# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） - - class TaskCreateStream(StateTransitionTask): @classmethod @@ -1775,60 +1766,25 @@ class TaskCreateStream(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create stream - - # CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); - stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) aggExpr = Dice.choice([ - 'count(*)', - 'avg(speed)', - # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable - 'sum(speed)', - 'stddev(speed)', - # SELECTOR functions - 'min(speed)', - 'max(speed)', - 'first(speed)', - 'last(speed)', - 'apercentile(speed, 10)', # TODO: TD-1316 - 'last_row(*)', # TODO: commented out per TD-3231, we should re-create - # Transformation Functions - 'twa(speed)' - - ]) # TODO: add more from 'top' + 'count(*)', 'avg(speed)', 'sum(speed)', 'stddev(speed)','min(speed)', 'max(speed)', 'first(speed)', 'last(speed)', + 'apercentile(speed, 10)', 'last_row(*)', 'twa(speed)']) + + stream_sql = '' # set default value if sub_tables: - - if sub_tables: # if not empty - sub_tbname = sub_tables[0] - # create stream with query above sub_table - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) - try: - self.execWtSql(wt, stream_sql) - Logging.debug("[OPS] stream is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # stream already exists - # stream need drop before drop table - pass - - else: - pass - + sub_tbname = sub_tables[0] + # create stream with query above sub_table + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.\ + format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) else: - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.\ + format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) - try: - self.execWtSql(wt, stream_sql) - Logging.debug("[OPS] stream is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # stream already exists - # stream need drop before drop table - pass class TaskCreateTopic(StateTransitionTask): @@ -1853,96 +1809,37 @@ class TaskCreateTopic(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create topic - # create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1; stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) - scalarExpr = Dice.choice([ '*','speed','color', - 'abs(speed)', - 'acos(speed)', - 'asin(speed)', - 'atan(speed)', - 'ceil(speed)', - 'cos(speed)', - 'cos(speed)', - 'floor(speed)', - 'log(speed,2)', - 'pow(speed,2)', - 'round(speed)', - 'sin(speed)', - 'sqrt(speed)', - 'char_length(color)', - 'concat(color,color)', - 'concat_ws(" ", color,color," ")', - 'length(color)', - 'lower(color)', - 'ltrim(color)', - 'substr(color , 2)', - 'upper(color)', - 'cast(speed as double)', - 'cast(ts as bigint)', - - ]) # TODO: add more from 'top' - if Dice.throw(3)==0: - if sub_tables: - - if sub_tables: # if not empty - sub_tbname = sub_tables[0] - # create stream with query above sub_table - topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0 ]: # topic already exists - # topic need drop before drop table - pass - - else: - pass - - else: + scalarExpr = Dice.choice([ '*','speed','color','abs(speed)','acos(speed)','asin(speed)','atan(speed)','ceil(speed)','cos(speed)','cos(speed)', + 'floor(speed)','log(speed,2)','pow(speed,2)','round(speed)','sin(speed)','sqrt(speed)','char_length(color)','concat(color,color)', + 'concat_ws(" ", color,color," ")','length(color)', 'lower(color)', 'ltrim(color)','substr(color , 2)','upper(color)','cast(speed as double)', + 'cast(ts as bigint)']) + topic_sql = '' # set default value + if Dice.throw(3)==0: # create topic : source data from sub query + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create topic : source data from sub query of sub stable + topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) + + else: # create topic : source data from sub query of stable topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] subquery topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass - elif Dice.throw(3)==1: + elif Dice.throw(3)==1: # create topic : source data from super table topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.queryWtSql(wt, topic_sql) - Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass - elif Dice.throw(3)==2: - topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] db topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass + + elif Dice.throw(3)==2: # create topic : source data from whole database + topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) else: pass + # exec create topics + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] db topic is creating at {}".format(time.time())) + class TaskDropTopics(StateTransitionTask): @classmethod @@ -1991,7 +1888,6 @@ class TaskDropStreams(StateTransitionTask): # tblName = sTable.getName() if sTable.hasStreams(wt.getDbConn()): sTable.dropStreams(wt.getDbConn()) # drop stream of database - # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable class TaskDropStreamTables(StateTransitionTask): @@ -2012,10 +1908,9 @@ class TaskDropStreamTables(StateTransitionTask): return sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place + wt.execSql("use db") # should always be in place # tblName = sTable.getName() if sTable.hasStreamTables(wt.getDbConn()): - # sTable.dropStreams(wt.getDbConn()) sTable.dropStreamTables(wt.getDbConn()) # drop stream tables class TaskCreateConsumers(StateTransitionTask): @@ -2031,13 +1926,9 @@ class TaskCreateConsumers(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): if Config.getConfig().connector_type == 'native': - dbname = self._db.getName() - + sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create Consumers - # if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task if sTable.hasTopics(wt.getDbConn()): sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) pass @@ -2088,30 +1979,13 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName - # if self.hasStreams(dbc): - # self.dropStreams(dbc) - # self.dropStreamTables(dbc) - # if self.hasTopics(dbc): - # self.dropTopics(dbName,None) - # self.dropTopics(dbName,self._stName) + try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table pass - # # stream need drop before drop table - # for stream in self.getStreamName(): - # drop_stream_sql = 'drop stream {}'.format(stream) - # try: - # dbc.execute(drop_stream_sql) - # except taos.error.ProgrammingError as err: - # # correcting for strange error number scheme - # errno3 = Helper.convertErrno(err.errno) - # if errno3 in [1011,0x3F3,0x03f3,0x2662,0x03f1]: # stream not exists - # pass - # dbc.execute("DROP TABLE {}".format(fullTableName)) - # pass else: if not skipCheck: @@ -2131,15 +2005,6 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: - # if self.hasStreams(dbc): - # self.dropStreams(dbc) - # self.dropStreamTables(dbc) - - # # drop topics before drop stables - # if self.hasTopics(dbc): - # self.dropTopics(dbc,self._dbName,None) - # self.dropTopics(dbc,self._dbName,self._stName ) - try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: @@ -2164,7 +2029,7 @@ class TdSuperTable: sql += " TAGS (dummy int) " dbc.execute(sql) - def createConsumer(self, dbc: DbConn , Consumer_nums): + def createConsumer(self, dbc,Consumer_nums): def generateConsumer(current_topic_list): conf = TaosTmqConf() @@ -2183,7 +2048,14 @@ class TdSuperTable: consumer.subscribe(topic_list) except TmqError as e : pass - time.sleep(5) # consumer work only 5 sec ,and then it will exit + + # consumer work only 30 sec + time_start = time.time() + while 1: + res = consumer.poll(1000) + if time.time() - time_start >5 : + break + # time.sleep(10) try: consumer.unsubscribe() except TmqError as e : @@ -2579,29 +2451,9 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - - # # drop streams before drop stables - # if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): - # self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) - # self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + - # # drop topics before drop stables - # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) - - - try: - self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) - except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = Helper.convertErrno(err.errno) - if (errno2 in [0x362]): # mnode invalid table name - isSuccess = False - Logging.debug("[DB] Acceptable error when dropping a table") - elif errno2 in [1011,0x3F3,0x03f3]: # table doesn't exist - - pass class TaskAlterTags(StateTransitionTask): From 4f9885a8594733c1ab1ab65b9129d39e1052a727 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Nov 2022 10:52:45 +0800 Subject: [PATCH 020/165] refact more code --- include/common/tdataformat.h | 10 ---------- source/common/src/tdataformat.c | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 5d0b012fca..1f53e731a5 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -74,7 +74,6 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // SValue ================================ -static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -280,15 +279,6 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); -static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); - } else { - memcpy(&pValue->val, p, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} - #endif #ifdef __cplusplus diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 73b887342c..b3d22d7ada 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1496,7 +1496,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); + memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); } From a5725b506c97fe25cc28bd8ebdb70d6d33bc09dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Nov 2022 17:22:09 +0800 Subject: [PATCH 021/165] more code --- include/common/tdataformat.h | 72 +++------ source/common/src/tdataformat.c | 175 +++++++++------------ source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 8 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 +- 5 files changed, 101 insertions(+), 162 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1f53e731a5..8038401867 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,17 +27,17 @@ extern "C" { #endif -typedef struct SBuffer SBuffer; -typedef struct SSchema SSchema; -typedef struct STColumn STColumn; -typedef struct STSchema STSchema; -typedef struct SValue SValue; -typedef struct SColVal SColVal; -typedef struct STSRow2 STSRow2; -typedef struct STSRowBuilder STSRowBuilder; -typedef struct STagVal STagVal; -typedef struct STag STag; -typedef struct SColData SColData; +typedef struct SBuffer SBuffer; +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct SValue SValue; +typedef struct SColVal SColVal; +typedef struct SRow SRow; +typedef struct SRowIter SRowIter; +typedef struct STagVal STagVal; +typedef struct STag STag; +typedef struct SColData SColData; #define HAS_NONE ((uint8_t)0x1) #define HAS_NULL ((uint8_t)0x2) @@ -70,8 +70,7 @@ int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); // STSchema ================================ -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); -void tTSchemaDestroy(STSchema *pTSchema); +void tTSchemaDestroy(STSchema *pTSchema); // SValue ================================ @@ -88,26 +87,12 @@ void tTSchemaDestroy(STSchema *pTSchema); #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) -// STSRow2 ================================ -#define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) -#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) +// SRow ================================ +int32_t tTSRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +void tTSRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tTSRowToArray(SRow *pRow, STSchema *pTSchema, SArray **ppArray); -int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); -int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); -void tTSRowFree(STSRow2 *pRow); -void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); -int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); -int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); - -// STSRowBuilder ================================ -#define tsRowBuilderInit() ((STSRowBuilder){0}) -#define tsRowBuilderClear(B) \ - do { \ - if ((B)->pBuf) { \ - taosMemoryFree((B)->pBuf); \ - } \ - } while (0) +// SRowIter ================================ // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); @@ -151,23 +136,12 @@ struct STSchema { STColumn columns[]; }; -#define TSROW_HAS_NONE ((uint8_t)0x1) -#define TSROW_HAS_NULL ((uint8_t)0x2U) -#define TSROW_HAS_VAL ((uint8_t)0x4U) -#define TSROW_KV_SMALL ((uint8_t)0x10U) -#define TSROW_KV_MID ((uint8_t)0x20U) -#define TSROW_KV_BIG ((uint8_t)0x40U) -#pragma pack(push, 1) -struct STSRow2 { - TSKEY ts; - uint8_t flags; - uint8_t data[]; -}; -#pragma pack(pop) - -struct STSRowBuilder { - int32_t szBuf; - uint8_t *pBuf; +struct SRow { + uint8_t flag; + uint8_t rsv; + uint16_t sver; + uint32_t len; + uint8_t data[]; }; struct SValue { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b3d22d7ada..6a676c51b7 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -65,44 +65,42 @@ static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { } } -// STSRow2 ======================================================================== -static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { - if (pb) { - switch (flags & 0xf) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NONE: - if (v) { - SET_BIT1(pb, idx, (uint8_t)1); - } else { - SET_BIT1(pb, idx, (uint8_t)0); - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL: - v = v - 1; - SET_BIT1(pb, idx, v); - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - SET_BIT2(pb, idx, v); - break; +// SRow ======================================================================== +int32_t tTSRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { + int32_t code = 0; - default: - ASSERT(0); + ASSERT(taosArrayGetSize(aColVal) == 0); + + uint8_t flag = 0; + int32_t iColVal = 0; + SColVal *pColVal = taosArrayGet(aColVal, iColVal); + int32_t iTColumn = 0; + STColumn *pTColumn = pTSchema->columns + iTColumn; + + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + flag |= HAS_VALUE; + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + flag |= HAS_NONE; + } else { // NULL + flag |= HAS_NULL; + } + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < taosArrayGetSize(aColVal)) ? taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid < pTColumn->colId) { + pColVal = (++iColVal < taosArrayGetSize(aColVal)) ? taosArrayGet(aColVal, iColVal) : NULL; + } else { // NONE + flag |= HAS_NONE; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } else { // NONE + flag |= HAS_NONE; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } -} -#define SET_IDX(p, i, n, f) \ - do { \ - if ((f)&TSROW_KV_SMALL) { \ - ((uint8_t *)(p))[i] = (n); \ - } else if ((f)&TSROW_KV_MID) { \ - ((uint16_t *)(p))[i] = (n); \ - } else { \ - ((uint32_t *)(p))[i] = (n); \ - } \ - } while (0) -#if 0 -int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { - int32_t code = 0; + #if 0 STColumn *pTColumn; SColVal *pColVal; @@ -238,7 +236,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S *ppRow = &pBuilder->tsRow; } else { // create a new one - *ppRow = (STSRow2 *)taosMemoryMalloc(sizeof(STSRow2)); + *ppRow = (SRow *)taosMemoryMalloc(sizeof(SRow)); if (*ppRow == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -391,33 +389,54 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S } #endif + _exit: return code; } -int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { - int32_t code = 0; - int32_t rLen = 0; +#if 0 +static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { + if (pb) { + switch (flags & 0xf) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NONE: + if (v) { + SET_BIT1(pb, idx, (uint8_t)1); + } else { + SET_BIT1(pb, idx, (uint8_t)0); + } + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL: + v = v - 1; + SET_BIT1(pb, idx, v); + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + SET_BIT2(pb, idx, v); + break; - TSROW_LEN(pRow, rLen); - (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen); - if (*ppRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + default: + ASSERT(0); + } } - memcpy(*ppRow, pRow, rLen); - -_exit: - return code; } +#define SET_IDX(p, i, n, f) \ + do { \ + if ((f)&TSROW_KV_SMALL) { \ + ((uint8_t *)(p))[i] = (n); \ + } else if ((f)&TSROW_KV_MID) { \ + ((uint16_t *)(p))[i] = (n); \ + } else { \ + ((uint32_t *)(p))[i] = (n); \ + } \ + } while (0) -void tTSRowFree(STSRow2 *pRow) { +void tTSRowFree(SRow *pRow) { if (pRow) { taosMemoryFree(pRow); } } -void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { +void tTSRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { uint8_t isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0; STColumn *pTColumn = &pTSchema->columns[iCol]; uint8_t flags = pRow->flags & (uint8_t)0xf; @@ -564,7 +583,7 @@ _return_value: return; } -int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { +int32_t tTSRowToArray(SRow *pRow, STSchema *pTSchema, SArray **ppArray) { int32_t code = 0; SColVal cv; @@ -584,62 +603,8 @@ _exit: return code; } #endif -int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { - int32_t n = 0; - - TSROW_LEN(pRow, n); - if (p) { - memcpy(p, pRow, n); - } - - return n; -} - -int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { - int32_t n = 0; - - *ppRow = (STSRow2 *)p; - TSROW_LEN(*ppRow, n); - - return n; -} // STSchema ======================================== -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { - *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); - if (*ppTSchema == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - (*ppTSchema)->numOfCols = ncols; - (*ppTSchema)->version = sver; - (*ppTSchema)->flen = 0; - (*ppTSchema)->vlen = 0; - (*ppTSchema)->tlen = 0; - - for (int32_t iCol = 0; iCol < ncols; iCol++) { - SSchema *pColumn = &pSchema[iCol]; - STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); - - pTColumn->colId = pColumn->colId; - pTColumn->type = pColumn->type; - pTColumn->flags = pColumn->flags; - pTColumn->bytes = pColumn->bytes; - pTColumn->offset = (*ppTSchema)->flen; - - // skip first column - if (iCol) { - (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; - if (IS_VAR_DATA_TYPE(pColumn->type)) { - (*ppTSchema)->vlen += (pColumn->bytes + 5); - } - } - } - - return 0; -} - void tTSchemaDestroy(STSchema *pTSchema) { if (pTSchema) taosMemoryFree(pTSchema); } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 518cfb62db..d335e84c4c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -112,8 +112,8 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // STSDBRowIter -void tRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); -SColVal *tRowIterNext(STSDBRowIter *pIter); +void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 87506126c7..b46a003638 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -596,20 +596,20 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; STSDBRowIter iter = {0}; - tRowIterInit(&iter, pRow, pTSchema); + tsdbRowIterInit(&iter, pRow, pTSchema); - SColVal *pColVal = tRowIterNext(&iter); + SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); while (pColVal && pColVal->cid < pDCBuilder->cid) { - pColVal = tRowIterNext(&iter); + pColVal = tsdbRowIterNext(&iter); } if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; - pColVal = tRowIterNext(&iter); + pColVal = tsdbRowIterNext(&iter); } else { code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); if (code) return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 3412e620c4..ae80af565b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -580,7 +580,7 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) { } // STSDBRowIter ====================================================== -void tRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { pIter->pRow = pRow; if (pRow->type == 0) { ASSERT(pTSchema); @@ -594,7 +594,7 @@ void tRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { } } -SColVal *tRowIterNext(STSDBRowIter *pIter) { +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); From e69821a64dd55197b3c50778d084062846ddbb4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Nov 2022 15:30:19 +0800 Subject: [PATCH 022/165] more code --- include/common/tdataformat.h | 6 +- source/common/src/tdataformat.c | 103 +++++++++++++++++++++++--------- 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8038401867..df0a479fcc 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -88,9 +88,9 @@ void tTSchemaDestroy(STSchema *pTSchema); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tTSRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); -void tTSRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tTSRowToArray(SRow *pRow, STSchema *pTSchema, SArray **ppArray); +int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP); // SRowIter ================================ diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 6a676c51b7..13dcf8ff3b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -46,37 +46,21 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); -#pragma pack(push, 1) -typedef struct { - int16_t nCols; - uint8_t idx[]; -} STSKVRow; -#pragma pack(pop) - -#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) - -// SValue -static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tPutBinary(p, pValue->pData, pValue->nData); - } else { - if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} - // SRow ======================================================================== -int32_t tTSRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { - int32_t code = 0; +#define ROW_FLG_KV ((uint8_t)0x10) - ASSERT(taosArrayGetSize(aColVal) == 0); +int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { + int32_t code = 0; uint8_t flag = 0; int32_t iColVal = 0; - SColVal *pColVal = taosArrayGet(aColVal, iColVal); + int32_t nColVal = (int32_t)taosArrayGetSize(aColVal); + SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, iColVal); int32_t iTColumn = 0; STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = sizeof(SRow); + int32_t nkv = sizeof(SRow); while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { @@ -88,12 +72,12 @@ int32_t tTSRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { flag |= HAS_NULL; } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < taosArrayGetSize(aColVal)) ? taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid < pTColumn->colId) { - pColVal = (++iColVal < taosArrayGetSize(aColVal)) ? taosArrayGet(aColVal, iColVal) : NULL; - } else { // NONE + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } } else { // NONE flag |= HAS_NONE; @@ -101,6 +85,71 @@ int32_t tTSRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } } + SRow *pRow = NULL; // todo + + iColVal = 0; + pColVal = (SColVal *)taosArrayGet(aColVal, iColVal); + iTColumn = 0; + pTColumn = pTSchema->columns + iTColumn; + if (ntp <= nkv) { // tuple row + pRow->flag = flag; + pRow->sver = pTSchema->version; + pRow->len = ntp; + + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + flag |= HAS_VALUE; + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + flag |= HAS_NONE; + } else { // NULL + flag |= HAS_NULL; + } + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + flag |= HAS_NONE; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + flag |= HAS_NONE; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + + } else { // kv row + pRow->flag = flag | ROW_FLG_KV; + pRow->sver = pTSchema->version; + pRow->len = ntp; + + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + flag |= HAS_VALUE; + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + flag |= HAS_NONE; + } else { // NULL + flag |= HAS_NULL; + } + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + flag |= HAS_NONE; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + flag |= HAS_NONE; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + } + #if 0 STColumn *pTColumn; SColVal *pColVal; From 31f667f83a438a5834944ba620960c4339a2c46b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 12:13:27 +0800 Subject: [PATCH 023/165] more code --- include/common/tdataformat.h | 1 + source/common/src/tdataformat.c | 235 ++++++++++++++++---------------- 2 files changed, 117 insertions(+), 119 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index df0a479fcc..770927203a 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -141,6 +141,7 @@ struct SRow { uint8_t rsv; uint16_t sver; uint32_t len; + TSKEY ts; uint8_t data[]; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 13dcf8ff3b..468092387f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -49,172 +49,160 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); // SRow ======================================================================== #define ROW_FLG_KV ((uint8_t)0x10) +#define KV_TYPE_LIT 0 +#define KV_TYPE_MID 1 +#define KV_TYPE_BIG 2 +typedef struct { + int8_t kvType; + int16_t nidx; + uint8_t idx[]; +} SKVIdx; + int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { int32_t code = 0; - uint8_t flag = 0; - int32_t iColVal = 0; - int32_t nColVal = (int32_t)taosArrayGetSize(aColVal); - SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, iColVal); - int32_t iTColumn = 0; - STColumn *pTColumn = pTSchema->columns + iTColumn; + ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); - int32_t ntp = sizeof(SRow); - int32_t nkv = sizeof(SRow); + // scan --------------- + uint8_t flag = 0; + int32_t iColVal = 1; + int32_t nColVal = taosArrayGetSize(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + int32_t iTColumn = 1; + STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = sizeof(SRow); + int32_t nkv = sizeof(SRow); + int32_t nidx = 0; while (pTColumn) { if (pColVal) { + ntp += TYPE_BYTES[pTColumn->type]; + if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + } else { + nkv = nkv + TYPE_BYTES[pTColumn->type]; + } + nidx++; } else if (COL_VAL_IS_NONE(pColVal)) { // NONE flag |= HAS_NONE; - } else { // NULL + } else if (COL_VAL_IS_NULL(pColVal)) { // NULL flag |= HAS_NULL; + nkv += tPutI16v(NULL, pTColumn->colId); + nidx++; + } else { + ASSERT(0); } + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; + ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } } else { // NONE flag |= HAS_NONE; + ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } - SRow *pRow = NULL; // todo - - iColVal = 0; - pColVal = (SColVal *)taosArrayGet(aColVal, iColVal); - iTColumn = 0; - pTColumn = pTSchema->columns + iTColumn; - if (ntp <= nkv) { // tuple row - pRow->flag = flag; - pRow->sver = pTSchema->version; - pRow->len = ntp; - - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - flag |= HAS_VALUE; - } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - flag |= HAS_NONE; - } else { // NULL - flag |= HAS_NULL; - } - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - flag |= HAS_NONE; - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } - } else { // NONE - flag |= HAS_NONE; - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + // compare --------------- + int32_t nRow = 0; + switch (flag) { + case HAS_NONE: + case HAS_NULL: + nRow = sizeof(SRow); + break; + case HAS_VALUE: + nRow = ntp; + break; + case HAS_NULL | HAS_NONE: + case HAS_VALUE | HAS_NONE: + case HAS_VALUE | HAS_NULL: + ntp = ntp + BIT1_SIZE(pTSchema->numOfCols); + nkv = nkv + sizeof(int32_t) * nidx; + if (ntp <= nkv) { + nRow = ntp; + } else { + nRow = nkv; + flag |= ROW_FLG_KV; } - } + break; + case HAS_VALUE | HAS_NULL | HAS_NONE: + ntp = ntp + BIT2_SIZE(pTSchema->numOfCols); + nkv = nkv + sizeof(int32_t) * nidx; + if (ntp <= nkv) { + nRow = ntp; + } else { + nRow = nkv; + flag |= ROW_FLG_KV; + } + break; + default: + ASSERT(0); + break; + } - } else { // kv row - pRow->flag = flag | ROW_FLG_KV; - pRow->sver = pTSchema->version; - pRow->len = ntp; + // alloc -------------- + SRow *pRow = NULL; + // build -------------- + pColVal = (SColVal *)taosArrayGet(aColVal, 0); + + pRow->flag = flag; + pRow->rsv = 0; + pRow->sver = pTSchema->version; + pRow->len = nRow; + memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); + + if (flag == HAS_NONE || flag == HAS_NULL) { + goto _exit; + } + + if (flag & ROW_FLG_KV) { // KV + } else { // TUPLE + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *pv = NULL; // todo + + iColVal = 1; + pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + iTColumn = 1; + pTColumn = pTSchema->columns + iTColumn; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - flag |= HAS_VALUE; - } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - flag |= HAS_NONE; - } else { // NULL - flag |= HAS_NULL; + if (COL_VAL_IS_VALUE(pColVal)) { + // todo + } else if (COL_VAL_IS_NONE(pColVal)) { + // todo + } else { + // todo } + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE - flag |= HAS_NONE; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } } else { // NONE - flag |= HAS_NONE; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } } #if 0 - STColumn *pTColumn; - SColVal *pColVal; - int32_t nColVal = taosArrayGetSize(pArray); - int32_t iColVal; - - ASSERT(nColVal > 0); - - // try - uint8_t flags = 0; - uint32_t ntv = 0; - uint32_t nkv = 0; - int16_t nTag = 0; - uint32_t maxIdx = 0; - - iColVal = 0; - for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - } else { - pColVal = NULL; - } - - if (iColumn == 0) { - ASSERT(pColVal->cid == pTColumn->colId); - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - - iColVal++; - } else { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - - if (COL_VAL_IS_NONE(pColVal)) { - flags |= TSROW_HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - flags |= TSROW_HAS_NULL; - maxIdx = nkv; - nTag++; - nkv += tPutI16v(NULL, -pTColumn->colId); - } else { - flags |= TSROW_HAS_VAL; - maxIdx = nkv; - nTag++; - nkv += tPutI16v(NULL, pTColumn->colId); - nkv += tPutValue(NULL, &pColVal->value, pTColumn->type); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ntv += tPutValue(NULL, &pColVal->value, pTColumn->type); - } - } - } else if (pColVal->cid > pTColumn->colId) { - flags |= TSROW_HAS_NONE; - } else { - ASSERT(0); - } - } else { - flags |= TSROW_HAS_NONE; - } - } - } - - ASSERT(flags); - // decide uint32_t nData = 0; uint32_t nDataT = 0; @@ -436,13 +424,22 @@ int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { continue; } } - #endif _exit: return code; } +void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + STColumn *pTColumn = pTSchema->columns + iCol; + + if (pRow->flag & ROW_FLG_KV) { // tuple row + // todo + } else { // kv row + // todo + } +} + #if 0 static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { if (pb) { From 78c570427953b52a5f74b337979c7d3ebb5020ba Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 12:17:57 +0800 Subject: [PATCH 024/165] more code --- source/common/src/tdataformat.c | 36 ++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 468092387f..0c1f971215 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -168,16 +168,42 @@ int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { goto _exit; } + iColVal = 1; + pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + iTColumn = 1; + pTColumn = pTSchema->columns + iTColumn; if (flag & ROW_FLG_KV) { // KV - } else { // TUPLE + SKVIdx *pIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; // todo + + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { + // todo + } else if (COL_VAL_IS_NONE(pColVal)) { + // todo + } else { + // todo + } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + + } else { // NONE + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + } else { // TUPLE uint8_t *pb = NULL; uint8_t *pf = NULL; uint8_t *pv = NULL; // todo - iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - iTColumn = 1; - pTColumn = pTSchema->columns + iTColumn; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { From dc2d5e3603acddaca17f73641f80d6fae4fb5f6d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 14:05:47 +0800 Subject: [PATCH 025/165] more code --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 100 ++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 11 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 770927203a..4cb4524bd3 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -88,7 +88,7 @@ void tTSchemaDestroy(STSchema *pTSchema); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0c1f971215..21ad6bcf6c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -58,7 +58,7 @@ typedef struct { uint8_t idx[]; } SKVIdx; -int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { int32_t code = 0; ASSERT(taosArrayGetSize(aColVal) > 0); @@ -153,7 +153,7 @@ int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // alloc -------------- - SRow *pRow = NULL; + SRow *pRow = NULL; // todo // build -------------- pColVal = (SColVal *)taosArrayGet(aColVal, 0); @@ -202,17 +202,32 @@ int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } else { // TUPLE uint8_t *pb = NULL; uint8_t *pf = NULL; - uint8_t *pv = NULL; // todo + uint8_t *pv = NULL; + int32_t nv = 0; + // todo + + // build impl while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { - // todo + // set bit map + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(int32_t *)(pf + pTColumn->offset) = nv; + nv += tPutU32v(pv + nv, pColVal->value.nData); + if (pColVal->value.nData) { + memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); + nv += pColVal->value.nData; + } + } else { + memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); + } } else if (COL_VAL_IS_NONE(pColVal)) { - // todo + // set bitmap } else { - // todo + // set bitmap } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -223,6 +238,7 @@ int32_t tRowPut(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } } else { // NONE + // set bitmap pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } @@ -457,15 +473,79 @@ _exit: } void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + ASSERT(iCol < pTSchema->numOfCols); + STColumn *pTColumn = pTSchema->columns + iCol; - if (pRow->flag & ROW_FLG_KV) { // tuple row - // todo - } else { // kv row - // todo + if (iCol == 0) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + return; + } + + if (pRow->flag == HAS_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } + + if (pRow->flag == HAS_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } + + if (pRow->flag & ROW_FLG_KV) { // KV Row + SKVIdx *pIdx = (SKVIdx *)pRow->data; + uint8_t *pv = pRow->data + sizeof(*pIdx) + sizeof(int32_t) * pIdx->nidx; + + int16_t lidx = 0; + int16_t ridx = pIdx->nidx - 1; + while (lidx <= ridx) { + int16_t mid = (lidx + ridx) >> 1; + uint8_t *pData = pv + pIdx->idx[mid]; // todo + int16_t cid = 0; + + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + } else { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData > 0) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pData, pTColumn->bytes); + } + } + return; + } else if (TABS(cid) < pTColumn->colId) { + lidx = mid + 1; + } else { + ridx = mid - 1; + } + } + + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + } else { // Tuple Row + if (pRow->flag == HAS_VALUE) { + } else { + // todo + } } } +int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP) { return 0; } + #if 0 static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { if (pb) { From e4df975a7faa36175837c681d3ef3206b6b02ec6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 14:33:50 +0800 Subject: [PATCH 026/165] more code --- source/common/src/tdataformat.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 21ad6bcf6c..49af4adb36 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -52,11 +52,14 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_TYPE_LIT 0 #define KV_TYPE_MID 1 #define KV_TYPE_BIG 2 + +#pragma pack(push, 1) typedef struct { int8_t kvType; int16_t nidx; - uint8_t idx[]; + char idx[]; } SKVIdx; +#pragma pack(pop) int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { int32_t code = 0; @@ -93,7 +96,7 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { flag |= HAS_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { // NULL flag |= HAS_NULL; - nkv += tPutI16v(NULL, pTColumn->colId); + nkv += tPutI16v(NULL, -pTColumn->colId); nidx++; } else { ASSERT(0); @@ -125,9 +128,11 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { case HAS_VALUE: nRow = ntp; break; - case HAS_NULL | HAS_NONE: - case HAS_VALUE | HAS_NONE: - case HAS_VALUE | HAS_NULL: + case (HAS_NULL | HAS_NONE): + // TODO + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): ntp = ntp + BIT1_SIZE(pTSchema->numOfCols); nkv = nkv + sizeof(int32_t) * nidx; if (ntp <= nkv) { @@ -137,7 +142,7 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { flag |= ROW_FLG_KV; } break; - case HAS_VALUE | HAS_NULL | HAS_NONE: + case (HAS_VALUE | HAS_NULL | HAS_NONE): ntp = ntp + BIT2_SIZE(pTSchema->numOfCols); nkv = nkv + sizeof(int32_t) * nidx; if (ntp <= nkv) { @@ -205,7 +210,9 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { uint8_t *pv = NULL; int32_t nv = 0; - // todo + if (flag == HAS_VALUE) { + } else if (flag == HAS_VALUE) { + } // build impl while (pTColumn) { From 04252283b9e9ca2bddc5144f3d4de23de1f595e6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 15:16:50 +0800 Subject: [PATCH 027/165] more code --- include/common/tdataformat.h | 4 + source/common/src/tdataformat.c | 330 ++++++++++---------------------- 2 files changed, 107 insertions(+), 227 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 4cb4524bd3..ea833b747c 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -93,6 +93,10 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP); // SRowIter ================================ +int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); +void tRowIterClose(SRowIter **ppIter); +SColVal *tRowIterNext(SRowIter *pIter); +SColVal *tRowIterNextNotNone(SRowIter *pIter); // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 49af4adb36..56e0f9fec7 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -56,7 +56,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #pragma pack(push, 1) typedef struct { int8_t kvType; - int16_t nidx; + int16_t nCol; char idx[]; } SKVIdx; #pragma pack(pop) @@ -251,230 +251,6 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } } -#if 0 - // decide - uint32_t nData = 0; - uint32_t nDataT = 0; - uint32_t nDataK = 0; - if (flags == TSROW_HAS_NONE || flags == TSROW_HAS_NULL) { - nData = 0; - } else { - switch (flags) { - case TSROW_HAS_VAL: - nDataT = pTSchema->flen + ntv; - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataT = BIT1_SIZE(pTSchema->numOfCols - 1); - break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - nDataT = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; - break; - default: - break; - ASSERT(0); - } - - uint8_t tflags = 0; - if (maxIdx <= UINT8_MAX) { - nDataK = sizeof(STSKVRow) + sizeof(uint8_t) * nTag + nkv; - tflags |= TSROW_KV_SMALL; - } else if (maxIdx <= UINT16_MAX) { - nDataK = sizeof(STSKVRow) + sizeof(uint16_t) * nTag + nkv; - tflags |= TSROW_KV_MID; - } else { - nDataK = sizeof(STSKVRow) + sizeof(uint32_t) * nTag + nkv; - tflags |= TSROW_KV_BIG; - } - - if (nDataT <= nDataK) { - nData = nDataT; - } else { - nData = nDataK; - flags |= tflags; - } - } - - // alloc - if (pBuilder) { - // create from a builder - if (nData == 0) { - pBuilder->tsRow.nData = 0; - pBuilder->tsRow.pData = NULL; - } else { - if (pBuilder->szBuf < nData) { - uint8_t *p = taosMemoryRealloc(pBuilder->pBuf, nData); - if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - pBuilder->pBuf = p; - pBuilder->szBuf = nData; - } - - pBuilder->tsRow.nData = nData; - pBuilder->tsRow.pData = pBuilder->pBuf; - } - - *ppRow = &pBuilder->tsRow; - } else { - // create a new one - *ppRow = (SRow *)taosMemoryMalloc(sizeof(SRow)); - if (*ppRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - if (nData == 0) { - (*ppRow)->nData = 0; - (*ppRow)->pData = NULL; - } else { - (*ppRow)->nData = nData; - (*ppRow)->pData = taosMemoryMalloc(nData); - if ((*ppRow)->pData == NULL) { - taosMemoryFree(*ppRow); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - } - - // build - (*ppRow)->flags = flags; - (*ppRow)->sver = pTSchema->version; - - pColVal = (SColVal *)taosArrayGet(pArray, 0); - (*ppRow)->ts = pColVal->value.ts; - - if ((*ppRow)->pData) { - STSKVRow *pTSKVRow = NULL; - uint8_t *pidx = NULL; - uint8_t *pkv = NULL; - uint8_t *pb = NULL; - uint8_t *pf = NULL; - uint8_t *ptv = NULL; - nkv = 0; - ntv = 0; - iColVal = 1; - - if ((flags & 0xf0) == 0) { - switch (flags & 0xf) { - case TSROW_HAS_VAL: - pf = (*ppRow)->pData; - ptv = pf + pTSchema->flen; - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - pb = (*ppRow)->pData; - break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - pb = (*ppRow)->pData; - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - ptv = pf + pTSchema->flen; - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - pb = (*ppRow)->pData; - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - ptv = pf + pTSchema->flen; - break; - default: - ASSERT(0); - break; - } - } else { - pTSKVRow = (STSKVRow *)(*ppRow)->pData; - pTSKVRow->nCols = 0; - pidx = pTSKVRow->idx; - if (flags & TSROW_KV_SMALL) { - pkv = pidx + sizeof(uint8_t) * nTag; - } else if (flags & TSROW_KV_MID) { - pkv = pidx + sizeof(uint16_t) * nTag; - } else { - pkv = pidx + sizeof(uint32_t) * nTag; - } - } - - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - } else { - pColVal = NULL; - } - - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - - if (COL_VAL_IS_NONE(pColVal)) { - goto _set_none; - } else if (COL_VAL_IS_NULL(pColVal)) { - goto _set_null; - } else { - goto _set_value; - } - } else if (pColVal->cid > pTColumn->colId) { - goto _set_none; - } else { - ASSERT(0); - } - } else { - goto _set_none; - } - - _set_none: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 0, iColumn - 1, flags); - if (flags & TSROW_HAS_VAL) { // set 0 - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; - } else { - tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); - } - } - } - continue; - - _set_null: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 1, iColumn - 1, flags); - if (flags & TSROW_HAS_VAL) { // set 0 - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; - } else { - tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); - } - } - } else { - SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); - pTSKVRow->nCols++; - nkv += tPutI16v(pkv + nkv, -pTColumn->colId); - } - continue; - - _set_value: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 2, iColumn - 1, flags); - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = ntv; - ntv += tPutValue(ptv + ntv, &pColVal->value, pTColumn->type); - } else { - tPutValue(pf + pTColumn->offset, &pColVal->value, pTColumn->type); - } - } else { - SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); - pTSKVRow->nCols++; - nkv += tPutI16v(pkv + nkv, pColVal->cid); - nkv += tPutValue(pkv + nkv, &pColVal->value, pTColumn->type); - } - continue; - } - } -#endif - _exit: return code; } @@ -504,10 +280,10 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { if (pRow->flag & ROW_FLG_KV) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; - uint8_t *pv = pRow->data + sizeof(*pIdx) + sizeof(int32_t) * pIdx->nidx; + uint8_t *pv = pRow->data + sizeof(*pIdx) + sizeof(int32_t) * pIdx->nCol; int16_t lidx = 0; - int16_t ridx = pIdx->nidx - 1; + int16_t ridx = pIdx->nCol - 1; while (lidx <= ridx) { int16_t mid = (lidx + ridx) >> 1; uint8_t *pData = pv + pIdx->idx[mid]; // todo @@ -553,6 +329,106 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP) { return 0; } +// SRowIter ======================================== +struct SRowIter { + SRow *pRow; + STSchema *pTSchema; + + int32_t iTColumn; + union { + struct { // kv + int32_t iCol; + uint8_t *pv; + }; + struct { // tupl + // todo + }; + }; + + SColVal cv; +}; + +int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { + int32_t code = 0; + + SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); + if (pIter == NULL) { + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _exit; + } + + pIter->pRow = pRow; + pIter->pTSchema = pTSchema; + +_exit: + if (code) { + *ppIter = NULL; + if (pIter) taosMemoryFree(pIter); + } else { + *ppIter = pIter; + } + return code; +} + +void tRowIterClose(SRowIter **ppIter) { + SRowIter *pIter = *ppIter; + if (pIter) { + taosMemoryFree(pIter); + } + *ppIter = NULL; +} + +SColVal *tRowIterNext(SRowIter *pIter) { + if (pIter->iTColumn >= pIter->pTSchema->numOfCols) { + return NULL; + } + + STColumn *pTColumn = pIter->pTSchema->columns + pIter->iTColumn; + + // timestamp + if (0 == pIter->iTColumn) { + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + goto _exit; + } + + if (pIter->pRow->flag == HAS_NONE) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } + + if (pIter->pRow->flag == HAS_NULL) { + pIter->cv = COL_VAL_NULL(pTColumn->type, pTColumn->colId); + goto _exit; + } + + if (pIter->pRow->flag & ROW_FLG_KV) { // KV + SKVIdx *pIdx = (SKVIdx *)pIter->pRow->data; + + while (pIter->iCol < pIdx->nCol) { + /* code */ + } + + // if (pIter->iIdx >= pIdx->nidx) { + // pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + // goto _exit; + // } + } else { // Tuple + // todo + } + +_exit: + pIter->iTColumn++; + return &pIter->cv; +} + +SColVal *tRowIterNextNotNone(SRowIter *pIter) { + // todo + return NULL; +} + #if 0 static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { if (pb) { From 8267cad1db8fd8d3a905fe89e9880271746a60e2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 15:32:55 +0800 Subject: [PATCH 028/165] more code --- source/common/src/tdataformat.c | 45 +++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 56e0f9fec7..ca16fb6584 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -338,6 +338,7 @@ struct SRowIter { union { struct { // kv int32_t iCol; + SKVIdx *pIdx; uint8_t *pv; }; struct { // tupl @@ -405,16 +406,44 @@ SColVal *tRowIterNext(SRowIter *pIter) { } if (pIter->pRow->flag & ROW_FLG_KV) { // KV - SKVIdx *pIdx = (SKVIdx *)pIter->pRow->data; + if (pIter->iCol < pIter->pIdx->nCol) { + uint8_t *pData = pIter->pv + pIter->pIdx->idx[pIter->iCol]; // todo + int16_t cid; - while (pIter->iCol < pIdx->nCol) { - /* code */ + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + } else { + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &pIter->cv.value.nData); + if (pIter->cv.value.nData > 0) { + pIter->cv.value.pData = pData; + } else { + pIter->cv.value.pData = NULL; + } + } else { + memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + } + } + + pIter->iCol++; + goto _exit; + } else if (TABS(cid) > pTColumn->colId) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } else { + ASSERT(0); + } + } else { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; } - - // if (pIter->iIdx >= pIdx->nidx) { - // pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - // goto _exit; - // } } else { // Tuple // todo } From b600b4c8bc088582acd2e7d6fe1fdf877c11fa24 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 21 Nov 2022 16:18:50 +0800 Subject: [PATCH 029/165] refactor(sync): modify wal pInfo++, to find which case can not pass --- source/libs/wal/src/walWrite.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 6cd127fed4..caae669e4a 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -325,7 +325,8 @@ int32_t walEndSnapshot(SWal *pWal) { SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); if (pInfo) { if (ver >= pInfo->lastVer) { - pInfo--; + //pInfo--; + pInfo++; } if (POINTER_DISTANCE(pInfo, pWal->fileInfoSet->pData) > 0) { wDebug("vgId:%d, wal end remove for %" PRId64, pWal->cfg.vgId, pInfo->firstVer); From 6d774dee4ed8000a15e4c686014a3cbd92109fa2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 16:19:10 +0800 Subject: [PATCH 030/165] enh: refact tmq pull request message --- source/common/src/tmsg.c | 78 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5dfcb62fad..08123e6f4a 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4723,6 +4723,84 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) { taosMemoryFreeClear(pReq->msg); } +int32_t tSerializeSTqOffsetVal(SEncoder *pEncoder, STqOffsetVal *pOffset) { + if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1; + + return 0; +} + +int32_t tDerializeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffset) { + if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1; + + return 0; +} + +int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeCStr(&encoder, pReq->subKey) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; + if (tEncodeI8(&encoder, pReq->useSnapshot) < 0) return -1; + if (tEncodeI32(&encoder, pReq->epoch) < 0) return -1; + if (tEncodeU64(&encoder, pReq->reqId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1; + if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeCStrTo(&decoder, pReq->subKey) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->useSnapshot) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->reqId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1; + if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { From 895b48f834489f7b4fcce60dbddf80c53bb5f1ac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 17:17:28 +0800 Subject: [PATCH 031/165] more code --- include/common/tdataformat.h | 1 - source/common/src/tdataformat.c | 73 ++++++++++++++++----------------- source/common/src/ttypes.c | 4 +- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ea833b747c..d29d1b8501 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -90,7 +90,6 @@ void tTSchemaDestroy(STSchema *pTSchema); // SRow ================================ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ca16fb6584..ff36039336 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -47,17 +47,14 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); // SRow ======================================================================== -#define ROW_FLG_KV ((uint8_t)0x10) - -#define KV_TYPE_LIT 0 -#define KV_TYPE_MID 1 -#define KV_TYPE_BIG 2 +#define KV_FLG_LIT ((uint8_t)0x10) +#define KV_FLG_MID ((uint8_t)0x20) +#define KV_FLG_BIG ((uint8_t)0x30) #pragma pack(push, 1) typedef struct { - int8_t kvType; int16_t nCol; - char idx[]; + char idx[]; // uint8_t * | uint16_t * | uint32_t * } SKVIdx; #pragma pack(pop) @@ -75,27 +72,29 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; int32_t iTColumn = 1; STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = sizeof(SRow); - int32_t nkv = sizeof(SRow); + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; int32_t nidx = 0; while (pTColumn) { if (pColVal) { - ntp += TYPE_BYTES[pTColumn->type]; - if (pColVal->cid == pTColumn->colId) { + ntp += TYPE_BYTES[pTColumn->type]; if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; + maxIdx = nkv; if (IS_VAR_DATA_TYPE(pTColumn->type)) { ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; } else { - nkv = nkv + TYPE_BYTES[pTColumn->type]; + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + TYPE_BYTES[pTColumn->type]; } nidx++; } else if (COL_VAL_IS_NONE(pColVal)) { // NONE flag |= HAS_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { // NULL flag |= HAS_NULL; + maxIdx = nkv; nkv += tPutI16v(NULL, -pTColumn->colId); nidx++; } else { @@ -119,43 +118,45 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // compare --------------- - int32_t nRow = 0; switch (flag) { case HAS_NONE: case HAS_NULL: - nRow = sizeof(SRow); + ntp = sizeof(SRow); break; case HAS_VALUE: - nRow = ntp; + ntp = sizeof(SRow) + ntp; break; case (HAS_NULL | HAS_NONE): - // TODO + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); break; case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - ntp = ntp + BIT1_SIZE(pTSchema->numOfCols); - nkv = nkv + sizeof(int32_t) * nidx; - if (ntp <= nkv) { - nRow = ntp; - } else { - nRow = nkv; - flag |= ROW_FLG_KV; - } + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = ntp + BIT2_SIZE(pTSchema->numOfCols); - nkv = nkv + sizeof(int32_t) * nidx; - if (ntp <= nkv) { - nRow = ntp; - } else { - nRow = nkv; - flag |= ROW_FLG_KV; - } + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; break; default: ASSERT(0); break; } + if (maxIdx <= UINT8_MAX) { + nkv = sizeof(SRow) + sizeof(SKVIdx) + sizeof(uint8_t) * nidx + nkv; + flag |= KV_FLG_LIT; + } else if (maxIdx <= UINT16_MAX) { + nkv = sizeof(SRow) + sizeof(SKVIdx) + sizeof(uint16_t) * nidx + nkv; + flag |= KV_FLG_MID; + } else { + nkv = sizeof(SRow) + sizeof(SKVIdx) + sizeof(uint32_t) * nidx + nkv; + flag |= KV_FLG_BIG; + } + int32_t nRow; + if (nkv < ntp) { + nRow = nkv; + } else { + nRow = ntp; + flag &= ((uint8_t)0x0f); + } // alloc -------------- SRow *pRow = NULL; // todo @@ -177,7 +178,7 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; iTColumn = 1; pTColumn = pTSchema->columns + iTColumn; - if (flag & ROW_FLG_KV) { // KV + if (flag & 0xf0) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; uint8_t *pv = NULL; // todo @@ -278,7 +279,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { return; } - if (pRow->flag & ROW_FLG_KV) { // KV Row + if (pRow->flag & 0xf0) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; uint8_t *pv = pRow->data + sizeof(*pIdx) + sizeof(int32_t) * pIdx->nCol; @@ -327,8 +328,6 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { } } -int32_t tRowToArr(SRow *pRow, STSchema *pTSchema, SArray **aColValP) { return 0; } - // SRowIter ======================================== struct SRowIter { SRow *pRow; @@ -405,7 +404,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } - if (pIter->pRow->flag & ROW_FLG_KV) { // KV + if (pIter->pRow->flag & 0xf0) { // KV if (pIter->iCol < pIter->pIdx->nCol) { uint8_t *pData = pIter->pv + pIter->pIdx->idx[pIter->iCol]; // todo int16_t cid; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 8c5d44b8d5..7b5d0a8805 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void *getDataMin(int32_t type, void* value) { +FORCE_INLINE void *getDataMin(int32_t type, void *value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMin; @@ -77,7 +77,7 @@ FORCE_INLINE void *getDataMin(int32_t type, void* value) { return value; } -FORCE_INLINE void *getDataMax(int32_t type, void* value) { +FORCE_INLINE void *getDataMax(int32_t type, void *value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMax; From de0703501589e7851c14741f1cb796bb85490f02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 17:41:17 +0800 Subject: [PATCH 032/165] more code --- include/common/tdataformat.h | 3 +- source/common/src/tdataformat.c | 75 +++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index d29d1b8501..179b46db0a 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -68,12 +68,11 @@ struct SBuffer { void tBufferDestroy(SBuffer *pBuffer); int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); +int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); // STSchema ================================ void tTSchemaDestroy(STSchema *pTSchema); -// SValue ================================ - // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_NONE ((int8_t)0x1) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ff36039336..763c791269 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,6 +43,16 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { return code; } +int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) { + int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); + if (code) return code; + + *ppData = pBuffer->pBuf + pBuffer->nBuf; + pBuffer->nBuf += nData; + + return code; +} + // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); @@ -66,16 +76,16 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nidx = 0; + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = taosArrayGetSize(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + int32_t iTColumn = 1; + STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { @@ -89,14 +99,14 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } else { nkv = nkv + tPutI16v(NULL, pTColumn->colId) + TYPE_BYTES[pTColumn->type]; } - nidx++; + nIdx++; } else if (COL_VAL_IS_NONE(pColVal)) { // NONE flag |= HAS_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { // NULL flag |= HAS_NULL; maxIdx = nkv; nkv += tPutI16v(NULL, -pTColumn->colId); - nidx++; + nIdx++; } else { ASSERT(0); } @@ -141,13 +151,13 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { break; } if (maxIdx <= UINT8_MAX) { - nkv = sizeof(SRow) + sizeof(SKVIdx) + sizeof(uint8_t) * nidx + nkv; + nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; flag |= KV_FLG_LIT; } else if (maxIdx <= UINT16_MAX) { - nkv = sizeof(SRow) + sizeof(SKVIdx) + sizeof(uint16_t) * nidx + nkv; + nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 1) + nkv; flag |= KV_FLG_MID; } else { - nkv = sizeof(SRow) + sizeof(SKVIdx) + sizeof(uint32_t) * nidx + nkv; + nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; flag |= KV_FLG_BIG; } int32_t nRow; @@ -159,7 +169,9 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // alloc -------------- - SRow *pRow = NULL; // todo + SRow *pRow = NULL; + code = tBufferReserve(pBuffer, nRow, (void **)&pRow); + if (code) return code; // build -------------- pColVal = (SColVal *)taosArrayGet(aColVal, 0); @@ -180,17 +192,35 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { pTColumn = pTSchema->columns + iTColumn; if (flag & 0xf0) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; - uint8_t *pv = NULL; // todo + int32_t iIdx = 0; + int32_t nv = 0; + uint8_t *pv = NULL; + if (flag & KV_FLG_LIT) { + pv = pIdx->idx + nIdx; + } else if (flag & KV_FLG_MID) { + pv = pIdx->idx + (nIdx << 1); + } else { + pv = pIdx->idx + (nIdx << 2); + } + pIdx->nCol = nIdx; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { - // todo - } else if (COL_VAL_IS_NONE(pColVal)) { - // todo - } else { - // todo + // offset = nv; todo + nv += tPutI16v(pv + nv, pTColumn->colId); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + nv += tPutU32v(pv + nv, pColVal->value.nData); + memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); + nv += pColVal->value.nData; + } else { + memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); + nv += pTColumn->bytes; + } + } else if (COL_VAL_IS_NULL(pColVal)) { + // offset = nv; todo + nv += tPutI16v(pv + nv, -pTColumn->colId); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -200,7 +230,6 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } else { pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } - } else { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } From c66390d4012bb81565a420cdc89e9d17481eef5d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 20:38:42 +0800 Subject: [PATCH 033/165] enh: refact tmq pull message --- include/common/tmsg.h | 4 +++ source/client/src/clientTmq.c | 44 +++++++++++++++----------- source/common/src/tmsg.c | 6 ++-- source/dnode/vnode/src/tq/tq.c | 56 +++++++++++++++++++--------------- 4 files changed, 65 insertions(+), 45 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 02d4c2279c..7d1c6a91b3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2948,6 +2948,10 @@ typedef struct { STqOffsetVal reqOffset; } SMqPollReq; +int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); +int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); + + typedef struct { int32_t vgId; int64_t offset; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 28b41b97c5..1dd3174c29 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1461,12 +1461,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return code; } -SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); - if (pReq == NULL) { - return NULL; - } - +void tmqBuildConsumeReqImpl(SMqPollReq *pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { /*strcpy(pReq->topic, pTopic->topicName);*/ /*strcpy(pReq->cgroup, tmq->groupId);*/ @@ -1485,9 +1480,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->useSnapshot = tmq->useSnapshot; - pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqPollReq)); - return pReq; + pReq->head.vgId = pVg->vgId; } SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { @@ -1559,15 +1552,32 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { #endif } atomic_store_32(&pVg->vgSkipCnt, 0); - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, timeout, pTopic, pVg); - if (pReq == NULL) { + + SMqPollReq req = {0}; + tmqBuildConsumeReqImpl(&req, tmq, timeout, pTopic, pVg); + int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); + if (msgSize < 0) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; } + char *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + tsem_post(&tmq->rspSem); + return -1; + } + + if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { + taosMemoryFree(msg); + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + tsem_post(&tmq->rspSem); + return -1; + } + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - taosMemoryFree(pReq); + taosMemoryFree(msg); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1581,7 +1591,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - taosMemoryFree(pReq); + taosMemoryFree(msg); taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); @@ -1589,11 +1599,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = sizeof(SMqPollReq), + .pData = msg, + .len = msgSize, .handle = NULL, }; - sendInfo->requestId = pReq->reqId; + sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqPollCb; @@ -1605,7 +1615,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { char offsetFormatBuf[80]; tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset); tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, - tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); + tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 08123e6f4a..811a492b87 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4766,7 +4766,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (buf != NULL) { SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); - pHead->vgId = htonl(pReq->header.vgId); + pHead->vgId = htonl(pReq->head.vgId); pHead->contLen = htonl(tlen + headLen); } @@ -4777,8 +4777,8 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t headLen = sizeof(SMsgHead); SMsgHead *pHead = buf; - pHead->vgId = pReq->header.vgId; - pHead->contLen = pReq->header.contLen; + pHead->vgId = pReq->head.vgId; + pHead->contLen = pReq->head.contLen; SDecoder decoder = {0}; tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 092128fe6e..1c543c434c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -458,20 +458,26 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { } int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { - SMqPollReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int32_t reqEpoch = pReq->epoch; + SMqPollReq req = {0}; int32_t code = 0; - STqOffsetVal reqOffset = pReq->reqOffset; STqOffsetVal fetchOffsetNew; SWalCkHead* pCkHead = NULL; + if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); + return -1; + } + + int64_t consumerId = req.consumerId; + int32_t reqEpoch = req.epoch; + STqOffsetVal reqOffset = req.reqOffset; + // 1.find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); /*ASSERT(pHandle);*/ if (pHandle == NULL) { tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId, - TD_VID(pTq->pVnode), pReq->subKey); + TD_VID(pTq->pVnode), req.subKey); return -1; } @@ -479,7 +485,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->consumerId != consumerId) { tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 ", in vgId:%d, subkey %s, handle consumer id %" PRId64, - consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; return -1; } @@ -493,13 +499,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId, - pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); + req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); // 2.reset offset if needed if (reqOffset.type > 0) { fetchOffsetNew = reqOffset; } else { - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); if (pOffset != NULL) { fetchOffsetNew = pOffset->val; char formatBuf[80]; @@ -508,7 +514,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { TD_VID(pTq->pVnode), formatBuf); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - if (pReq->useSnapshot) { + if (req.useSnapshot) { if (pHandle->fetchMeta) { tqOffsetResetToMeta(&fetchOffsetNew, 0); } else { @@ -520,21 +526,21 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version); - if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { code = -1; } tDeleteSMqDataRsp(&dataRsp); return code; } else { STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pReq); + tqInitTaosxRsp(&taosxRsp, &req); tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -543,7 +549,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64 " in vg %d, subkey %s, reset none failed", - pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey); + pHandle->subKey, consumerId, TD_VID(pTq->pVnode), req.subKey); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return -1; } @@ -552,7 +558,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); // lock taosWLockLatch(&pTq->pushLock); tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew); @@ -580,7 +586,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { #endif taosWUnLockLatch(&pTq->pushLock); - if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { code = -1; } @@ -599,13 +605,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pReq); + tqInitTaosxRsp(&taosxRsp, &req); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); if (metaRsp.metaRspLen > 0) { - if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { code = -1; } tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, send meta offset type:%d,uid:%" PRId64 @@ -618,7 +624,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } if (taosxRsp.blockNum > 0) { - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -648,13 +654,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (consumerEpoch > reqEpoch) { tqWarn("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, vg %d offset %" PRId64 ", found new consumer epoch %d, discard req epoch %d", - consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); + consumerId, req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); break; } if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -665,7 +671,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SWalCont* pHead = &pCkHead->head; tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, - pReq->epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); + req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; @@ -675,7 +681,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -693,7 +699,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { code = -1; taosMemoryFreeClear(pCkHead); tDeleteSTaosxRsp(&taosxRsp); From dcec02fd5765517bd808dde01e1582c77c6f00ba Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Nov 2022 22:04:57 +0800 Subject: [PATCH 034/165] more code --- source/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 763c791269..6f0082aa6d 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -97,7 +97,7 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; } else { - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + TYPE_BYTES[pTColumn->type]; + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; } nIdx++; } else if (COL_VAL_IS_NONE(pColVal)) { // NONE From 6d5a2567749750728111f6ad029c921d21ca893c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Nov 2022 22:40:20 +0800 Subject: [PATCH 035/165] fix(query): fix some memory leaks. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 26 +++++++++---- source/libs/executor/src/cachescanoperator.c | 1 + source/libs/executor/src/executorimpl.c | 40 -------------------- 3 files changed, 19 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 986cba8b17..fbafc63382 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -361,11 +361,17 @@ static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; + pInfo->iter.hasVal = false; pInfo->iiter.hasVal = false; + if (pInfo->iter.iter != NULL) { pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); } + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); pInfo->lastKey = ts; } @@ -373,6 +379,8 @@ static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { static void clearBlockScanInfo(STableBlockScanInfo* p) { p->iterInit = false; + + p->iter.hasVal = false; p->iiter.hasVal = false; if (p->iter.iter != NULL) { @@ -388,9 +396,9 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { tMapDataClear(&p->mapData); } -static void destroyAllBlockScanInfo(SHashObj* pTableMap, bool clearEntry) { +static void destroyAllBlockScanInfo(SHashObj* pTableMap) { void* p = NULL; - while (clearEntry && ((p = taosHashIterate(pTableMap, p)) != NULL)) { + while ((p = taosHashIterate(pTableMap, p)) != NULL) { clearBlockScanInfo(*(STableBlockScanInfo**)p); } @@ -2226,6 +2234,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea if (pReader->pReadSnap->pMem != NULL) { d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { + ASSERT(pBlockScanInfo->iter.iter == NULL); code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); @@ -3789,9 +3798,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); } - STsdbReader* p = pReader->innerReader[0] != NULL ? pReader->innerReader[0] : pReader; - - pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList, numOfTables); + pReader->status.pTableMap = createDataBlockScanInfo(pReader, pTableList, numOfTables); if (pReader->status.pTableMap == NULL) { tsdbReaderClose(pReader); *ppReader = NULL; @@ -3849,7 +3856,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } { - if (pReader->innerReader[0] != NULL) { + if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { STsdbReader* p = pReader->innerReader[0]; p->status.pTableMap = NULL; @@ -3887,9 +3894,12 @@ void tsdbReaderClose(STsdbReader* pReader) { cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); - destroyAllBlockScanInfo(pReader->status.pTableMap, (pReader->innerReader[0] == NULL) ? true : false); + if (pReader->status.pTableMap != NULL) { + destroyAllBlockScanInfo(pReader->status.pTableMap); + clearBlockScanInfoBuf(&pReader->blockInfoBuf); + } + blockDataDestroy(pReader->pResBlock); - clearBlockScanInfoBuf(&pReader->blockInfoBuf); if (pReader->pFileReader != NULL) { tsdbDataFReaderClose(&pReader->pFileReader); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index b78ba8ac0a..922ed05653 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -246,6 +246,7 @@ void destroyLastrowScanOperator(void* param) { pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); } + cleanupExprSupp(&pInfo->pseudoExprSup); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0dd5765aa4..a7e955100c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -20,7 +20,6 @@ #include "querynodes.h" #include "tfill.h" #include "tname.h" -#include "tref.h" #include "tdatablock.h" #include "tglobal.h" @@ -134,45 +133,6 @@ static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); -#if 0 -static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, char* pData, - int16_t bytes, bool masterscan, uint64_t uid) { - bool existed = false; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - - SResultRow** p1 = - (SResultRow**)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - - // in case of repeat scan/reverse scan, no new time window added. - if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { - if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return p1 != NULL; - } - - if (p1 != NULL) { - if (pResultRowInfo->size == 0) { - existed = false; - } else if (pResultRowInfo->size == 1) { - // existed = (pResultRowInfo->pResult[0] == (*p1)); - } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); - int64_t* index = - taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - existed = true; - } else { - existed = false; - } - } - } - - return existed; - } - - return p1 != NULL; -} -#endif - SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) { SFilePage* pData = NULL; From ae0bd036a07fa0d0a36e12aba9b0ae5d6c98d061 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 23:11:00 +0800 Subject: [PATCH 036/165] fix: more log --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 24240e82ef..4aa07cad98 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -148,8 +148,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pHead->vgId, pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), qtype); + dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(), + TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); return terrno != 0 ? terrno : -1; } From bb51d3cac774def54dda8ec7a5eefee8f7626333 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 21 Nov 2022 23:11:00 +0800 Subject: [PATCH 037/165] fix: fix error for select count(c2),count(ts) from table caused by invalid ts column aggregation --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 986cba8b17..bb2de2ec93 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4118,9 +4118,13 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } else if (pAgg->colId < pSup->colIds[j]) { i += 1; } else if (pSup->colIds[j] < pAgg->colId) { + if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { + taosArrayPush(pNewAggList, &pSup->tsColAgg); + } else { // all date in this block are null - SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; - taosArrayPush(pNewAggList, &nullColAgg); + SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; + taosArrayPush(pNewAggList, &nullColAgg); + } j += 1; } } From 514ad921c74b4d20c818340304db65b574b70e04 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 22 Nov 2022 01:16:38 +0800 Subject: [PATCH 038/165] test:add testcase of enterprise installPackages --- packaging/MPtestJenkinsfile | 197 +++++++++++++++++++++++------------- packaging/testpackage.sh | 107 +++++++++++++++++--- 2 files changed, 218 insertions(+), 86 deletions(-) diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 5dc8024f29..0e958e8384 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -40,27 +40,32 @@ pipeline { choice( name: 'sourcePath', choices: ['nas','web'], - description: 'choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' + description: 'Choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' + ) + choice( + name: 'verMode', + choices: ['all','community','enterprise'], + description: 'Choice which types of package you want do check ' ) string ( name:'version', - defaultValue:'3.0.1.6', - description: 'release version number,eg: 3.0.0.1 or 3.0.0.' + defaultValue:'3.0.1.7', + description: 'Release version number,eg: 3.0.0.1 or 3.0.0.' ) string ( name:'baseVersion', - defaultValue:'3.0.1.6', - description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' + defaultValue:'3.0.1.7', + description: 'The number of baseVerison is generally not modified.Now it is 3.0.0.1' ) string ( name:'toolsVersion', defaultValue:'2.2.7', - description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' + description: 'Release version number,eg:2.2.0' ) string ( name:'toolsBaseVersion', defaultValue:'2.1.2', - description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' + description: 'The number of baseVerison is generally not modified.Now it is 2.1.2' ) } environment{ @@ -68,10 +73,10 @@ pipeline { TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' BRANCH_NAME = '3.0' - - TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz" + + TD_SERVER_TAR = "${preServerPackag}-${version}-Linux-x64.tar.gz" BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz" - + TD_SERVER_ARM_TAR = "TDengine-server-${version}-Linux-arm64.tar.gz" BASE_TD_SERVER_ARM_TAR = "TDengine-server-${baseVersion}-Linux-arm64.tar.gz" @@ -108,19 +113,24 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sync_source("${BRANCH_NAME}") sh ''' + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done + cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb + python3 checkPackageRuning.py ''' } } @@ -131,22 +141,26 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sync_source("${BRANCH_NAME}") sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - dpkg -r tdengine - ''' + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done + + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb + python3 checkPackageRuning.py + dpkg -r tdengine + ''' } } } @@ -156,19 +170,24 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sync_source("${BRANCH_NAME}") sh ''' + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done + cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm + python3 checkPackageRuning.py ''' } } @@ -179,21 +198,26 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sync_source("${BRANCH_NAME}") sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done + cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm + python3 checkPackageRuning.py sudo rpm -e tdengine - ''' + ''' } } } @@ -203,9 +227,16 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sync_source("${BRANCH_NAME}") sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${sourcePath} - python3 checkPackageRuning.py + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done ''' } } @@ -219,8 +250,16 @@ pipeline { steps { timeout(time: 30, unit: 'MINUTES'){ sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${sourcePath} + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done python3 checkPackageRuning.py 192.168.0.21 ''' } @@ -231,8 +270,16 @@ pipeline { steps { timeout(time: 30, unit: 'MINUTES'){ sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${sourcePath} + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f client -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done python3 checkPackageRuning.py 192.168.0.24 ''' } @@ -245,8 +292,16 @@ pipeline { steps { timeout(time: 30, unit: 'MINUTES'){ sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client ${sourcePath} + if [ "${verMode}" = "all" ];then + verMode="community enterprise" + fi + verModeList=${verMode} + for verModeSin in ${verModeList} + do + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f client -l true -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + done python3 checkPackageRuning.py 192.168.0.21 ''' } diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index b0e33835dd..846c8d160f 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -1,15 +1,72 @@ #!/bin/sh + + +function usage() { + echo "$0" + echo -e "\t -f test file type,server/client/tools/" + echo -e "\t -m pacakage version Type,community/enterprise" + echo -e "\t -l package type,lite or not" + echo -e "\t -c operation type,x64/arm64" + echo -e "\t -v pacakage version,3.0.1.7" + echo -e "\t -o pacakage version,3.0.1.7" + echo -e "\t -s source Path,web/nas" + echo -e "\t -t package Type,tar/rpm/deb" + echo -e "\t -h help" +} + + #parameter scriptDir=$(dirname $(readlink -f $0)) -packgeName=$1 -version=$2 -originPackageName=$3 -originversion=$4 -testFile=$5 -# sourcePath:web/nas -sourcePath=$6 +version="3.0.1.7" +originversion="3.0.1.7" +testFile="server" +verMode="communtity" +sourcePath="nas" +cpuType="x64" +lite="true" +packageType="tar" subFile="taos.tar.gz" +while getopts "m:c:f:l:s:o:t:v:h" opt; do + case $opt in + m) + verMode=$OPTARG + ;; + v) + version=$OPTARG + ;; + f) + testFile=$OPTARG + ;; + l) + lite=$OPTARG + ;; + s) + sourcePath=$OPTARG + ;; + o) + originversion=$OPTARG + ;; + c) + cpuType=$OPTARG + ;; + t) + packageType=$OPTARG + ;; + h) + usage + exit 0 + ;; + ?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + + +echo "testFile:${testFile},verMode:${verMode},lite:${lite},cpuType:${cpuType},packageType:${packageType},version-${version},originversion:${originversion},sourcePath:${sourcePath}" # Color setting RED='\033[41;30m' GREEN='\033[1;32m' @@ -21,20 +78,40 @@ BLUE_DARK='\033[0;34m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -if [ ${testFile} = "server" ];then - tdPath="TDengine-server-${version}" - originTdpPath="TDengine-server-${originversion}" +if [[ ${verMode} = "enterprise" ]];then + prePackag="TDengine-enterprise-${testFile}" +elif [ ${verMode} = "community" ];then + prePackag="TDengine-${testFile}" +fi +if [ ${lite} = "true" ];then + packageLite="-Lite" +elif [ ${lite} = "false" ];then + packageLite="" +fi +if [[ "$packageType" = "tar" ]] ;then + packageType="tar.gz" +fi + +tdPath="${prePackag}-${version}" +originTdpPath="${prePackag}-${originversion}" + +packgeName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" +originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" + +if [ "$testFile" == "server" ] ;then installCmd="install.sh" elif [ ${testFile} = "client" ];then - tdPath="TDengine-client-${version}" - originTdpPath="TDengine-client-${originversion}" installCmd="install_client.sh" elif [ ${testFile} = "tools" ];then tdPath="taosTools-${version}" originTdpPath="taosTools-${originversion}" + packgeName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" + originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" installCmd="install-taostools.sh" fi + +echo "tdPath:${tdPath},originTdpPath:${originTdpPath},packgeName:${packgeName},originPackageName:${originPackageName}" function cmdInstall { command=$1 if command -v ${command} ;then @@ -76,16 +153,16 @@ file=$1 versionPath=$2 sourceP=$3 nasServerIP="192.168.1.131" -packagePath="/nas/TDengine/v${versionPath}/community" +packagePath="/nas/TDengine/v${versionPath}/${verMode}" if [ -f ${file} ];then echoColor YD "${file} already exists ,it will delete it and download it again " rm -rf ${file} fi -if [ ${sourceP} = 'web' ];then +if [[ ${sourceP} = 'web' ]];then echoColor BD "====download====:wget https://www.taosdata.com/assets-download/3.0/${file}" wget https://www.taosdata.com/assets-download/3.0/${file} -elif [ ${sourceP} = 'nas' ];then +elif [[ ${sourceP} = 'nas' ]];then echoColor BD "====download====:scp root@${nasServerIP}:${packagePath}/${file} ." scp root@${nasServerIP}:${packagePath}/${file} . fi From c3514a97a204275d034d612cae41c322762d85c3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 22 Nov 2022 01:53:08 +0800 Subject: [PATCH 039/165] test:modify two options that downloads the installPackages --- packaging/MPtestJenkinsfile | 93 ++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 0e958e8384..de6cace508 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -117,19 +117,26 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar python3 checkPackageRuning.py - - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar - python3 checkPackageRuning.py done + ''' + sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb + bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + ''' + + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb python3 checkPackageRuning.py ''' } @@ -145,19 +152,26 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar python3 checkPackageRuning.py - - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar - python3 checkPackageRuning.py done + ''' + sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb + bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + ''' + + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb python3 checkPackageRuning.py dpkg -r tdengine ''' @@ -174,19 +188,25 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar - python3 checkPackageRuning.py - - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar python3 checkPackageRuning.py done + ''' + sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm + bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + ''' + + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm python3 checkPackageRuning.py ''' } @@ -202,19 +222,25 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar python3 checkPackageRuning.py - - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar - python3 checkPackageRuning.py - done + done + ''' + sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm + bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py + ''' + + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm python3 checkPackageRuning.py sudo rpm -e tdengine ''' @@ -231,6 +257,9 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -254,6 +283,8 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -270,16 +301,10 @@ pipeline { steps { timeout(time: 30, unit: 'MINUTES'){ sh ''' - if [ "${verMode}" = "all" ];then - verMode="community enterprise" - fi - verModeList=${verMode} - for verModeSin in ${verModeList} - do - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f client -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar - python3 checkPackageRuning.py - done + verModeList=community + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh -f server -m ${verModeSin} -f client -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 checkPackageRuning.py python3 checkPackageRuning.py 192.168.0.24 ''' } @@ -296,10 +321,12 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} + ''' + sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh -f server -m ${verModeSin} -f client -l true -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + bash testpackage.sh -f server -m ${verModeSin} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar python3 checkPackageRuning.py done python3 checkPackageRuning.py 192.168.0.21 From 642202681c7ee1c4359d794b7589f9059a1af5b1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 09:22:13 +0800 Subject: [PATCH 040/165] test: add asan case --- tests/parallel_test/cases.task | 132 ++++++++++++++++----------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 75ff515078..1741e282fb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -431,40 +431,40 @@ ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py ,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py -,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -,,,system-test,python3 ./test.py -f 2-query/abs.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py -,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -,,,system-test,python3 ./test.py -f 2-query/arccos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -,,,system-test,python3 ./test.py -f 2-query/arcsin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -,,,system-test,python3 ./test.py -f 2-query/arctan.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -R ,,,system-test,python3 ./test.py -f 2-query/avg.py ,,,system-test,python3 ./test.py -f 2-query/avg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -,,,system-test,python3 ./test.py -f 2-query/between.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -,,,system-test,python3 ./test.py -f 2-query/bottom.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -,,,system-test,python3 ./test.py -f 2-query/cast.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -,,,system-test,python3 ./test.py -f 2-query/ceil.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -,,,system-test,python3 ./test.py -f 2-query/char_length.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -R ,,,system-test,python3 ./test.py -f 2-query/concat.py ,,,system-test,python3 ./test.py -f 2-query/concat.py -R ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py @@ -472,63 +472,63 @@ ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -,,,system-test,python3 ./test.py -f 2-query/cos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -,,,system-test,python3 ./test.py -f 2-query/count_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -,,,system-test,python3 ./test.py -f 2-query/count.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py -,,,system-test,python3 ./test.py -f 2-query/db.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -,,,system-test,python3 ./test.py -f 2-query/diff.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -,,,system-test,python3 ./test.py -f 2-query/distinct.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -,,,system-test,python3 ./test.py -f 2-query/explain.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -,,,system-test,python3 ./test.py -f 2-query/first.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -,,,system-test,python3 ./test.py -f 2-query/floor.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -,,,system-test,python3 ./test.py -f 2-query/function_null.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py -,,,system-test,python3 ./test.py -f 2-query/histogram.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -R ,,,system-test,python3 ./test.py -f 2-query/interp.py ,,,system-test,python3 ./test.py -f 2-query/interp.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -,,,system-test,python3 ./test.py -f 2-query/irate.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -,,,system-test,python3 ./test.py -f 2-query/join.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -R ,,,system-test,python3 ./test.py -f 2-query/last_row.py ,,,system-test,python3 ./test.py -f 2-query/last_row.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -,,,system-test,python3 ./test.py -f 2-query/last.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -,,,system-test,python3 ./test.py -f 2-query/length.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R ,,,system-test,python3 ./test.py -f 2-query/lower.py @@ -536,25 +536,25 @@ ,,,system-test,python3 ./test.py -f 2-query/ltrim.py ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -,,,system-test,python3 ./test.py -f 2-query/mavg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -,,,system-test,python3 ./test.py -f 2-query/max.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -,,,system-test,python3 ./test.py -f 2-query/min.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -R ,,,system-test,python3 ./test.py -f 2-query/mode.py ,,,system-test,python3 ./test.py -f 2-query/mode.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -,,,system-test,python3 ./test.py -f 2-query/Now.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -,,,system-test,python3 ./test.py -f 2-query/percentile.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -,,,system-test,python3 ./test.py -f 2-query/pow.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -,,,system-test,python3 ./test.py -f 2-query/round.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -R ,,,system-test,python3 ./test.py -f 2-query/rtrim.py ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py @@ -566,51 +566,51 @@ ,,,system-test,python3 ./test.py -f 2-query/sml.py ,,,system-test,python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -,,,system-test,python3 ./test.py -f 2-query/spread.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -,,,system-test,python3 ./test.py -f 2-query/sqrt.py -R +,,y,system-test,./pytest.sh python3./test.py -f 2-query/sqrt.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -,,,system-test,python3 ./test.py -f 2-query/statecount.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py ,,,system-test,python3 ./test.py -f 2-query/stateduration.py -R ,,,system-test,python3 ./test.py -f 2-query/substr.py ,,,system-test,python3 ./test.py -f 2-query/substr.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py -,,,system-test,python3 ./test.py -f 2-query/sum.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py -R ,,,system-test,python3 ./test.py -f 2-query/tail.py ,,,system-test,python3 ./test.py -f 2-query/tail.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -,,,system-test,python3 ./test.py -f 2-query/tan.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -,,,system-test,python3 ./test.py -f 2-query/Timediff.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -,,,system-test,python3 ./test.py -f 2-query/timezone.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -,,,system-test,python3 ./test.py -f 2-query/Today.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -,,,system-test,python3 ./test.py -f 2-query/top.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -R ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -,,,system-test,python3 ./test.py -f 2-query/twa.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R ,,,system-test,python3 ./test.py -f 2-query/union.py ,,,system-test,python3 ./test.py -f 2-query/union.py -R ,,,system-test,python3 ./test.py -f 2-query/unique.py ,,,system-test,python3 ./test.py -f 2-query/unique.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -,,,system-test,python3 ./test.py -f 2-query/upper.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -,,,system-test,python3 ./test.py -f 2-query/varchar.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -,,,system-test,python3 ./test.py -f 2-query/case_when.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R ,,,system-test,python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py From 726d4d69aad90bca7d517aa1a14d4de4d902677e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 09:55:36 +0800 Subject: [PATCH 041/165] test: add asan case --- tests/parallel_test/cases.task | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1741e282fb..b91d53bcf2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -572,7 +572,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -,,,system-test,python3 ./test.py -f 2-query/stateduration.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -R ,,,system-test,python3 ./test.py -f 2-query/substr.py ,,,system-test,python3 ./test.py -f 2-query/substr.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py @@ -601,8 +601,8 @@ ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R -,,,system-test,python3 ./test.py -f 2-query/union.py -,,,system-test,python3 ./test.py -f 2-query/union.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -R ,,,system-test,python3 ./test.py -f 2-query/unique.py ,,,system-test,python3 ./test.py -f 2-query/unique.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py From cafc5e4cae25d659a08d58d565c99e05bbdd9507 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 22 Nov 2022 10:43:56 +0800 Subject: [PATCH 042/165] enh: refact fetch message --- include/common/tmsg.h | 5 +- source/common/src/tmsg.c | 55 +++++++++++++++++++++ source/libs/executor/src/exchangeoperator.c | 37 +++++++++----- source/libs/qworker/src/qwMsg.c | 19 +++---- source/libs/scheduler/src/schRemote.c | 27 ++++++---- 5 files changed, 109 insertions(+), 34 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7d1c6a91b3..29d68aba14 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1629,7 +1629,6 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); void tFreeSSubQueryMsg(SSubQueryMsg *pReq); - typedef struct { SMsgHead header; uint64_t sId; @@ -1667,6 +1666,10 @@ typedef struct { int32_t execId; } SResFetchReq; +int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); +int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); + + typedef struct { SMsgHead header; uint64_t sId; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c6ec4cabe5..7575554bcf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4723,6 +4723,61 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) { taosMemoryFreeClear(pReq->msg); } + +int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->execId) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + int32_t tSerializeSTqOffsetVal(SEncoder *pEncoder, STqOffsetVal *pOffset) { if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index c57a1b38eb..ffbdc1362d 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -403,27 +403,42 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas loadRemoteDataCallback(pWrapper, &pBuf, code); taosMemoryFree(pWrapper); } else { - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { + SResFetchReq req = {0}; + req.header.vgId = pSource->addr.nodeId; + req.sId = pSource->schedId; + req.taskId = pSource->taskId; + req.queryId = pTaskInfo->id.queryId; + req.execId = pSource->execId; + + int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); + if (msgSize < 0) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } + + void* msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + return pTaskInfo->code; + } + + if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + taosMemoryFree(msg); + return pTaskInfo->code; + } qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources); - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); + taosMemoryFreeClear(msg); taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -432,8 +447,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas pMsgSendInfo->param = pWrapper; pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgType = pSource->fetchMsgType; pMsgSendInfo->fp = loadRemoteDataCallback; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index bb8a7cd140..d9a7cea411 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -499,27 +499,22 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int return TSDB_CODE_QRY_INVALID_INPUT; } - SResFetchReq *msg = pMsg->pCont; + SResFetchReq req = {0}; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1); - if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + if (tDeserializeSResFetchReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + QW_ELOG("tDeserializeSResFetchReq %d failed", pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); - msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->execId = ntohl(msg->execId); - - uint64_t sId = msg->sId; - uint64_t qId = msg->queryId; - uint64_t tId = msg->taskId; + uint64_t sId = req.sId; + uint64_t qId = req.queryId; + uint64_t tId = req.taskId; int64_t rId = 0; - int32_t eId = msg->execId; + int32_t eId = req.execId; SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connInfo = pMsg->info, .msgType = pMsg->msgType}; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b0bc0df850..c154060d21 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1083,22 +1083,29 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_SCH_FETCH: case TDMT_SCH_MERGE_FETCH: { - msgSize = sizeof(SResFetchReq); + SResFetchReq req = {0}; + req.header.vgId = addr->nodeId; + req.sId = schMgmt.sId; + req.queryId = pJob->queryId; + req.taskId = pTask->taskId; + req.execId = pTask->execId; + + msgSize = tSerializeSResFetchReq(NULL, 0, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSResFetchReq get size, msgSize:%d", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResFetchReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->execId = htonl(pTask->execId); - + if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { + SCH_TASK_ELOG("tSerializeSResFetchReq %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } break; } case TDMT_SCH_DROP_TASK: { From b9f7d181b64255a913a06d29abecb7727324a501 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Nov 2022 10:47:50 +0800 Subject: [PATCH 043/165] fix: free the batch create table reqs' decoded comment field --- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5c8c166833..c75d1ffded 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -609,6 +609,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR _exit: for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; + taosMemoryFree(pCreateReq->comment); taosArrayDestroy(pCreateReq->ctb.tagName); } taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); From fbb0a1e921dbc12aff0e3abdaae054d603834512 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Nov 2022 11:08:55 +0800 Subject: [PATCH 044/165] fix(query): set correct tsdbreader during the creation of tableblockscan info --- source/dnode/vnode/src/tsdb/tsdbRead.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fbafc63382..69fae4d481 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -329,20 +329,6 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } taosHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); - -#if 0 -// STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; - if (ASCENDING_TRAVERSE(pTsdbReader->order)) { - int64_t skey = pTsdbReader->window.skey; - info.lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; - } else { - int64_t ekey = pTsdbReader->window.ekey; - info.lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; - } - - taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); -#endif - tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, pScanInfo->lastKey, pTsdbReader->idStr); } @@ -3798,7 +3784,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); } - pReader->status.pTableMap = createDataBlockScanInfo(pReader, pTableList, numOfTables); + STsdbReader* p = (pReader->innerReader[0] != NULL)? pReader->innerReader[0]:pReader; + pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList, numOfTables); if (pReader->status.pTableMap == NULL) { tsdbReaderClose(pReader); *ppReader = NULL; From 96f691cd5c04ac2eb32cdd13e992891962d21dd9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Nov 2022 11:12:20 +0800 Subject: [PATCH 045/165] avoid invalid error --- source/common/src/tglobal.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 27dcbd5be3..f2d8b9aa7c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -277,7 +277,9 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; - if (taosGetFqdn(defaultFqdn) != 0) return -1; + if (taosGetFqdn(defaultFqdn) != 0) { + strcpy(defaultFqdn, "localhost"); + } if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; From ce59e4fff4266715ceec0209407a15760e06aec5 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Tue, 22 Nov 2022 11:45:25 +0800 Subject: [PATCH 046/165] add test case --- tests/parallel_test/cases.task | 2 + tests/system-test/1-insert/drop.py | 128 +++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tests/system-test/1-insert/drop.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a9a02d8380..7b06fa5eab 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -616,6 +616,8 @@ ,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py ,,,system-test,python3 ./test.py -f 1-insert/keep_expired.py +,,,system-test,python3 ./test.py -f 1-insert/drop.py +,,,system-test,python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 ,,,system-test,python3 ./test.py -f 2-query/join2.py ,,,system-test,python3 ./test.py -f 2-query/union1.py ,,,system-test,python3 ./test.py -f 2-query/concat2.py diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py new file mode 100644 index 0000000000..d511d1aa7a --- /dev/null +++ b/tests/system-test/1-insert/drop.py @@ -0,0 +1,128 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +import math +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.dbname = 'db' + self.ntbname = f"{self.dbname}.ntb" + self.rowNum = 10 + self.tbnum = 20 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'binary(20)', + 'col13': 'nchar(20)' + } + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def drop_ntb_check(self): + tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + for k,v in self.column_dict.items(): + if v.lower() == "timestamp": + tdSql.query(f'select * from {self.ntbname} where {k} = {self.ts}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {self.ntbname}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + for k,v in self.column_dict.items(): + if v.lower() == "timestamp": + tdSql.query(f'select * from {self.ntbname} where {k} = {self.ts}') + tdSql.checkRows(1) + tdSql.execute(f'drop database {self.dbname}') + + def drop_stb_ctb_check(self): + stbname = f'{self.dbname}.{tdCom.getLongName(5,"letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar}") + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) + for k,v in self.column_dict.items(): + for i in range(self.tbnum): + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname}_{i} where {k} = {self.ts}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {stbname}_{i}') + tdSql.execute(f'flush database {self.dbname}') + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) + for k,v in self.column_dict.items(): + for i in range(self.tbnum): + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname}_{i} where {k} = {self.ts}') + tdSql.checkRows(1) + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname} where {k} = {self.ts}') + tdSql.checkRows(self.tbnum) + tdSql.execute(f'drop table {stbname}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) + for k,v in self.column_dict.items(): + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname} where {k} = {self.ts}') + tdSql.checkRows(self.tbnum) + tdSql.execute(f'drop database {self.dbname}') + + def run(self): + self.drop_ntb_check() + self.drop_stb_ctb_check() + pass + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 3f5a87a5b12d1773afd5c51305a57f49cd76cb92 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Nov 2022 11:59:09 +0800 Subject: [PATCH 047/165] more code --- source/common/src/tdataformat.c | 89 +++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 6f0082aa6d..9caee778ad 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -61,6 +61,10 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_BIG ((uint8_t)0x30) +#define ROW_BIT_NONE ((uint8_t)0x0) +#define ROW_BIT_NULL ((uint8_t)0x1) +#define ROW_BIT_VALUE ((uint8_t)0x2) + #pragma pack(push, 1) typedef struct { int16_t nCol; @@ -68,6 +72,29 @@ typedef struct { } SKVIdx; #pragma pack(pop) +#define ROW_SET_BITMAP(PB, FLAG, IDX, VAL) \ + do { \ + if (PB) { \ + switch (FLAG) { \ + case (HAS_NULL | HAS_NONE): \ + SET_BIT1(PB, IDX, VAL); \ + break; \ + case (HAS_VALUE | HAS_NONE): \ + SET_BIT1(PB, IDX, (VAL) ? (VAL)-1 : 0); \ + break; \ + case (HAS_VALUE | HAS_NULL): \ + SET_BIT1(PB, IDX, (VAL)-1); \ + break; \ + case (HAS_VALUE | HAS_NULL | HAS_NONE): \ + SET_BIT2(PB, IDX, VAL); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ + } \ + } while (0) + int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { int32_t code = 0; @@ -208,7 +235,15 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { - // offset = nv; todo + if (flag & KV_FLG_LIT) { + ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; + } else if (flag & KV_FLG_MID) { + ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; + } else { + ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; + } + iIdx++; + nv += tPutI16v(pv + nv, pTColumn->colId); if (IS_VAR_DATA_TYPE(pTColumn->type)) { nv += tPutU32v(pv + nv, pColVal->value.nData); @@ -219,7 +254,14 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { nv += pTColumn->bytes; } } else if (COL_VAL_IS_NULL(pColVal)) { - // offset = nv; todo + if (flag & KV_FLG_LIT) { + ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; + } else if (flag & KV_FLG_MID) { + ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; + } else { + ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; + } + iIdx++; nv += tPutI16v(pv + nv, -pTColumn->colId); } @@ -240,16 +282,36 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { uint8_t *pv = NULL; int32_t nv = 0; - if (flag == HAS_VALUE) { - } else if (flag == HAS_VALUE) { + switch (flag) { + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; } // build impl while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { - // set bit map + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; @@ -261,21 +323,26 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } else { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } - } else if (COL_VAL_IS_NONE(pColVal)) { - // set bitmap - } else { - // set bitmap + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + } else { // NULL + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; } } else { // NONE - // set bitmap + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } From 33538e39e24715cbb8f68497a9eaad86dcc9edba Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 12:21:18 +0800 Subject: [PATCH 048/165] test: add asan case --- tests/parallel_test/cases.task | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b91d53bcf2..1e5f2fad44 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -568,7 +568,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -,,y,system-test,./pytest.sh python3./test.py -f 2-query/sqrt.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py @@ -597,8 +597,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -R ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R -,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py From 1d48426e94ddb732bc132dc8d17e0262598fcbd6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 22 Nov 2022 12:45:31 +0800 Subject: [PATCH 049/165] refactor(sync): del wal in multi-replicas --- include/libs/sync/sync.h | 2 +- source/libs/sync/src/syncAppendEntriesReply.c | 3 ++ source/libs/sync/src/syncMain.c | 49 +++++++++++++------ source/libs/sync/src/syncRaftLog.c | 12 ++++- source/libs/sync/src/syncTimeout.c | 24 ++++----- source/libs/sync/src/syncUtil.c | 4 +- 6 files changed, 63 insertions(+), 31 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 3a808ac6f3..513ba8cb34 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -36,7 +36,7 @@ extern "C" { #define SYNC_DEL_WAL_MS (1000 * 60) #define SYNC_ADD_QUORUM_COUNT 3 #define SYNC_MNODE_LOG_RETENTION 10000 -#define SYNC_VNODE_LOG_RETENTION 100 +#define SYNC_VNODE_LOG_RETENTION 20 #define SNAPSHOT_MAX_CLOCK_SKEW_MS 1000 * 10 #define SNAPSHOT_WAIT_MS 1000 * 30 diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 13ea250155..c602788b19 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -67,6 +67,9 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (pMsg->matchIndex > oldMatchIndex) { syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); syncMaybeAdvanceCommitIndex(ths); + + // maybe update minMatchIndex + ths->minMatchIndex = syncMinMatchIndex(ths); } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a427d7aa0c..88b8ba7e25 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -243,6 +243,18 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { goto _DEL_WAL; } else { + lastApplyIndex -= SYNC_VNODE_LOG_RETENTION; + + SyncIndex beginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + SyncIndex endIndex = pSyncNode->pLogStore->syncLogEndIndex(pSyncNode->pLogStore); + bool isEmpty = pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore); + + if (isEmpty || !(lastApplyIndex >= beginIndex && lastApplyIndex <= endIndex)) { + sNTrace(pSyncNode, "new-snapshot-index:%" PRId64 ", empty:%d, do not delete wal", lastApplyIndex, isEmpty); + syncNodeRelease(pSyncNode); + return 0; + } + // vnode if (pSyncNode->replicaNum > 1) { // multi replicas @@ -300,26 +312,31 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { _DEL_WAL: do { - SyncIndex snapshottingIndex = atomic_load_64(&pSyncNode->snapshottingIndex); + SSyncLogStoreData* pData = pSyncNode->pLogStore->data; + SyncIndex snapshotVer = walGetSnapshotVer(pData->pWal); + SyncIndex walCommitVer = walGetCommittedVer(pData->pWal); + SyncIndex wallastVer = walGetLastVer(pData->pWal); + if (lastApplyIndex <= walCommitVer) { + SyncIndex snapshottingIndex = atomic_load_64(&pSyncNode->snapshottingIndex); - if (snapshottingIndex == SYNC_INDEX_INVALID) { - atomic_store_64(&pSyncNode->snapshottingIndex, lastApplyIndex); - pSyncNode->snapshottingTime = taosGetTimestampMs(); + if (snapshottingIndex == SYNC_INDEX_INVALID) { + atomic_store_64(&pSyncNode->snapshottingIndex, lastApplyIndex); + pSyncNode->snapshottingTime = taosGetTimestampMs(); + + code = walBeginSnapshot(pData->pWal, lastApplyIndex); + if (code == 0) { + sNTrace(pSyncNode, "wal snapshot begin, index:%" PRId64 ", last apply index:%" PRId64, + pSyncNode->snapshottingIndex, lastApplyIndex); + } else { + sNError(pSyncNode, "wal snapshot begin error since:%s, index:%" PRId64 ", last apply index:%" PRId64, + terrstr(terrno), pSyncNode->snapshottingIndex, lastApplyIndex); + atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID); + } - SSyncLogStoreData* pData = pSyncNode->pLogStore->data; - code = walBeginSnapshot(pData->pWal, lastApplyIndex); - if (code == 0) { - sNTrace(pSyncNode, "wal snapshot begin, index:%" PRId64 ", last apply index:%" PRId64, - pSyncNode->snapshottingIndex, lastApplyIndex); } else { - sNError(pSyncNode, "wal snapshot begin error since:%s, index:%" PRId64 ", last apply index:%" PRId64, - terrstr(terrno), pSyncNode->snapshottingIndex, lastApplyIndex); - atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID); + sNTrace(pSyncNode, "snapshotting for %" PRId64 ", do not delete wal for new-snapshot-index:%" PRId64, + snapshottingIndex, lastApplyIndex); } - - } else { - sNTrace(pSyncNode, "snapshotting for %" PRId64 ", do not delete wal for new-snapshot-index:%" PRId64, - snapshottingIndex, lastApplyIndex); } } while (0); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index db0b6d1d02..2f824b6b3b 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -375,7 +375,17 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - // ASSERT(walCommit(pWal, index) == 0); + + // need not update + SyncIndex snapshotVer = walGetSnapshotVer(pWal); + SyncIndex walCommitVer = walGetCommittedVer(pWal); + SyncIndex wallastVer = walGetLastVer(pWal); + + if (index < snapshotVer || index > wallastVer) { + // ignore + return 0; + } + int32_t code = walCommit(pWal, index); if (code != 0) { int32_t err = terrno; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 3d4583aadb..151e5cdf46 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -62,18 +62,20 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { syncNodeCleanConfigIndex(ths); } - // end timeout wal snapshot int64_t timeNow = taosGetTimestampMs(); - if (timeNow - ths->snapshottingIndex > SYNC_DEL_WAL_MS && - atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { - SSyncLogStoreData* pData = ths->pLogStore->data; - int32_t code = walEndSnapshot(pData->pWal); - if (code != 0) { - sNError(ths, "timer wal snapshot end error since:%s", terrstr()); - return -1; - } else { - sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); - atomic_store_64(&ths->snapshottingIndex, SYNC_INDEX_INVALID); + if (atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { + // end timeout wal snapshot + if (timeNow - ths->snapshottingTime > SYNC_DEL_WAL_MS && + atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { + SSyncLogStoreData* pData = ths->pLogStore->data; + int32_t code = walEndSnapshot(pData->pWal); + if (code != 0) { + sNError(ths, "timer wal snapshot end error since:%s", terrstr()); + return -1; + } else { + sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); + atomic_store_64(&ths->snapshottingIndex, SYNC_INDEX_INVALID); + } } } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index b50336cd63..4fc7dd245d 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -239,11 +239,11 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo "vgId:%d, sync %s " "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, bch:%d, r-num:%d, lcfg:%" PRId64 + ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - pNode->pRaftCfg->isStandBy, aqItems, pNode->pRaftCfg->batchSize, pNode->replicaNum, + pNode->pRaftCfg->isStandBy, aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); } From 88f244bca5fe0124cf9235e8eeb70fc68bdc9fc1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Nov 2022 12:59:27 +0800 Subject: [PATCH 050/165] more code --- source/common/src/tdataformat.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 9caee778ad..d464517f7e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -377,15 +377,29 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { if (pRow->flag & 0xf0) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; - uint8_t *pv = pRow->data + sizeof(*pIdx) + sizeof(int32_t) * pIdx->nCol; + uint8_t *pv = NULL; + if (pRow->flag & KV_FLG_LIT) { + pv = pIdx->idx + pIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pIdx->idx + (pIdx->nCol << 1); + } else { + pv = pIdx->idx + (pIdx->nCol << 2); + } int16_t lidx = 0; int16_t ridx = pIdx->nCol - 1; while (lidx <= ridx) { int16_t mid = (lidx + ridx) >> 1; - uint8_t *pData = pv + pIdx->idx[mid]; // todo - int16_t cid = 0; + uint8_t *pData = NULL; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pIdx->idx)[mid]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pIdx->idx)[mid]; + } else { + pData = pv + ((uint32_t *)pIdx->idx)[mid]; + } + int16_t cid; pData += tGetI16v(pData, &cid); if (TABS(cid) == pTColumn->colId) { From cd49e5b4e5252791bcb99dfdda6436ab1011276c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 22 Nov 2022 13:40:25 +0800 Subject: [PATCH 051/165] test:add testcase of enterprise installPackages --- packaging/MPtestJenkinsfile | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index de6cace508..dad5b7f129 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -116,10 +116,7 @@ pipeline { if [ "${verMode}" = "all" ];then verMode="community enterprise" fi - verModeList=${verMode} - ''' - - sh ''' + verModeList=${verMode} for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -152,9 +149,6 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} - ''' - - sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -188,8 +182,6 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} - ''' - sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -222,8 +214,6 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} - ''' - sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -257,9 +247,6 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} - ''' - - sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -283,8 +270,6 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} - ''' - sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging @@ -321,8 +306,6 @@ pipeline { verMode="community enterprise" fi verModeList=${verMode} - ''' - sh ''' for verModeSin in ${verModeList} do cd ${TDENGINE_ROOT_DIR}/packaging From 1a3e429d10481c3a2920acb489e5a79d96d839ac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Nov 2022 13:54:27 +0800 Subject: [PATCH 052/165] more code --- source/common/src/tdataformat.c | 56 +++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d464517f7e..fb9d3d1624 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -354,6 +354,7 @@ _exit: void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { ASSERT(iCol < pTSchema->numOfCols); + ASSERT(pRow->sver == pTSchema->version); STColumn *pTColumn = pTSchema->columns + iCol; @@ -431,9 +432,60 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } else { // Tuple Row - if (pRow->flag == HAS_VALUE) { + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t bv = ROW_BIT_VALUE; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + break; + } + + if (bv == ROW_BIT_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } else if (bv == ROW_BIT_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } + + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } } else { - // todo + memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); } } } From 4d4ba0ff110781487e8e01a877b5fffbee6b16fb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Nov 2022 13:55:11 +0800 Subject: [PATCH 053/165] docs: update jdbc version in connector matrix for3.0 (#18332) * docs: update csharp connector status * docs: fix csharp ws bulk pulling * docs: clarify database param is optional to websocket dsn * docs: fix python version and a few typos * docs: fix jdbc version in connector matrix --- docs/en/14-reference/03-connector/index.mdx | 15 +++++++-------- docs/zh/08-connector/index.md | 15 +++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 4fd9c452d8..54031db618 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -26,14 +26,13 @@ Using REST connection can support a broader range of operating systems as it doe TDengine version updates often add new features, and the connector versions in the list are the best-fit versions of the connector. -| **TDengine Versions** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| --------------------- | -------- | ---------- | ------------ | ------------- | --------------- | -------- | -| **3.0.0.0 and later** | 3.0.0 | current version | 3.0 branch | 3.0.0 | 3.0.0 | current version | -| **2.4.0.14 and up** | 2.0.38 | current version | develop branch | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | current version | -| **2.4.0.6 and up** | 2.0.37 | current version | develop branch | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | current version | -| **2.4.0.4 - 2.4.0.5** | 2.0.37 | current version | develop branch | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | current version | -| **2.2.x.x ** | 2.0.36 | current version | master branch | n/a | 2.0.7 - 2.0.9 | current version | -| **2.0.x.x ** | 2.0.34 | current version | master branch | n/a | 2.0.1 - 2.0.6 | current version | +| **TDengine Versions** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | +| --------------------------- | -------------- | -------------- | -------------- | ------------- | --------------- | --------------- | +| **3.0.0.0 and later** | 3.0.2 + | latest version | 3.0 branch | 3.0.0 | 3.0.0 | current version | +| **2.4.0.14 and up ** | 2.0.38 | latest version | develop branch | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | current version | +| **2.4.0.4 - 2.4.0.13 ** | 2.0.37 | latest version | develop branch | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | current version | +| **2.2.x.x ** | 2.0.36 | latest version | master branch | n/a | 2.0.7 - 2.0.9 | current version | +| **2.0.x.x ** | 2.0.34 | latest version | master branch | n/a | 2.0.1 - 2.0.6 | current version | ## Functional Features diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index e00e0b2fa2..eecf564b90 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -26,14 +26,13 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 TDengine 版本更新往往会增加新的功能特性，列表中的连接器版本为连接器最佳适配版本。 -| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| --------------------- | -------- | ---------- | ------------ | ------------- | --------------- | -------- | -| **3.0.0.0 及以上** | 3.0.0 | 当前版本 | 3.0 分支 | 3.0.0 | 3.0.0 | 当前版本 | -| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.4.0.6 及以上** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.4.0.4 - 2.4.0.5** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | -| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | +| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | +| ---------------------- | --------- | ---------- | ------------ | ------------- | --------------- | -------- | +| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.0.0 | 当前版本 | +| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | +| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | +| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | +| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | ## 功能特性 From 886e84c3150199058b6f6f2955b2d6d87dc63759 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Tue, 22 Nov 2022 13:56:35 +0800 Subject: [PATCH 054/165] enh(taosAdapter): keep debug information (#18324) --- tools/CMakeLists.txt | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 0f3c383b9e..d61d25602b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -101,19 +101,20 @@ ELSE () BUILD_COMMAND COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib - COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" - COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" +# COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" +# COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND cmake -E echo "Comparessing taosadapter.exe" - COMMAND cmake -E time upx taosadapter.exe +# COMMAND cmake -E echo "Comparessing taosadapter.exe" +# COMMAND cmake -E time upx taosadapter.exe COMMAND cmake -E echo "Copy taosadapter.exe" COMMAND cmake -E copy taosadapter.exe ${CMAKE_BINARY_DIR}/build/bin/taosadapter.exe COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ - COMMAND cmake -E echo "Copy taosadapter-debug.exe" - COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin +# COMMAND cmake -E echo "Copy taosadapter-debug.exe" +# COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin ) ELSE (TD_WINDOWS) MESSAGE("Building taosAdapter on non-Windows") @@ -128,19 +129,20 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" +# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" +# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND cmake -E echo "Comparessing taosadapter.exe" - COMMAND upx taosadapter || : +# COMMAND cmake -E echo "Comparessing taosadapter.exe" +# COMMAND upx taosadapter || : COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ - COMMAND cmake -E echo "Copy taosadapter-debug" - COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin +# COMMAND cmake -E echo "Copy taosadapter-debug" +# COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) ENDIF (TD_WINDOWS) ENDIF () From d3fe6f0ba38a04ff68c8c65330c5dbc54c170e83 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 22 Nov 2022 14:36:27 +0800 Subject: [PATCH 055/165] fix [ASAN] memory leak while tdbRealloc --- source/libs/executor/src/timewindowoperator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fc78f8a20d..1ec7d6d26b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2703,6 +2703,7 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S pChildSup->rowEntryInfoOffset, &pChInfo->aggSup); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + releaseOutputBuf(pChInfo->pState, pWinRes, pChResult); } if (num > 0 && pUpdatedMap) { saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pUpdatedMap); From fe3cda4f8741b542d3cc1600fa469fd798f229a9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Nov 2022 14:57:22 +0800 Subject: [PATCH 056/165] more code --- source/common/src/tdataformat.c | 322 +++++++++++--------------------- 1 file changed, 104 insertions(+), 218 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fb9d3d1624..44ececee58 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -498,19 +498,22 @@ struct SRowIter { int32_t iTColumn; union { struct { // kv - int32_t iCol; - SKVIdx *pIdx; - uint8_t *pv; + int32_t iCol; + SKVIdx *pIdx; }; - struct { // tupl - // todo + struct { // tuple + uint8_t *pb; + uint8_t *pf; }; + uint8_t *pv; }; SColVal cv; }; int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { + ASSERT(pRow->sver == pTSchema->version); + int32_t code = 0; SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); @@ -521,6 +524,45 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { pIter->pRow = pRow; pIter->pTSchema = pTSchema; + pIter->iTColumn = 0; + + if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; + + if (pRow->flag & 0xf0) { + pIter->iCol = 0; + pIter->pIdx = (SKVIdx *)pRow->data; + if (pRow->flag & KV_FLG_LIT) { + pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + } else { + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + } + } else { + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + pIter->pb = pRow->data; + break; + case HAS_VALUE: + pIter->pf = pRow->data; + pIter->pv = pIter->pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pIter->pb = pRow->data; + pIter->pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pIter->pv = pIter->pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pIter->pb = pRow->data; + pIter->pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pIter->pv = pIter->pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } + } _exit: if (code) { @@ -568,9 +610,17 @@ SColVal *tRowIterNext(SRowIter *pIter) { if (pIter->pRow->flag & 0xf0) { // KV if (pIter->iCol < pIter->pIdx->nCol) { - uint8_t *pData = pIter->pv + pIter->pIdx->idx[pIter->iCol]; // todo - int16_t cid; + uint8_t *pData; + if (pIter->pRow->flag & KV_FLG_LIT) { + pData = pIter->pv + ((uint8_t *)pIter->pIdx->idx)[pIter->iCol]; + } else if (pIter->pRow->flag & KV_FLG_MID) { + pData = pIter->pv + ((uint16_t *)pIter->pIdx->idx)[pIter->iCol]; + } else { + pData = pIter->pv + ((uint32_t *)pIter->pIdx->idx)[pIter->iCol]; + } + + int16_t cid; pData += tGetI16v(pData, &cid); if (TABS(cid) == pTColumn->colId) { @@ -606,7 +656,53 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } } else { // Tuple - // todo + uint8_t bv = ROW_BIT_VALUE; + if (pIter->pb) { + switch (pIter->pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); + break; + case HAS_VALUE: + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); + if (bv) bv++; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1); + break; + default: + ASSERT(0); + break; + } + + if (bv == ROW_BIT_NONE) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } else if (bv == ROW_BIT_NULL) { + pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + goto _exit; + } + } + + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); + pData += tGetU32v(pData, &pIter->cv.value.nData); + if (pIter->cv.value.nData > 0) { + pIter->cv.value.pData = pData; + } else { + pIter->cv.value.pData = NULL; + } + } else { + memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + } + goto _exit; } _exit: @@ -619,216 +715,6 @@ SColVal *tRowIterNextNotNone(SRowIter *pIter) { return NULL; } -#if 0 -static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { - if (pb) { - switch (flags & 0xf) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NONE: - if (v) { - SET_BIT1(pb, idx, (uint8_t)1); - } else { - SET_BIT1(pb, idx, (uint8_t)0); - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL: - v = v - 1; - SET_BIT1(pb, idx, v); - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - SET_BIT2(pb, idx, v); - break; - - default: - ASSERT(0); - } - } -} -#define SET_IDX(p, i, n, f) \ - do { \ - if ((f)&TSROW_KV_SMALL) { \ - ((uint8_t *)(p))[i] = (n); \ - } else if ((f)&TSROW_KV_MID) { \ - ((uint16_t *)(p))[i] = (n); \ - } else { \ - ((uint32_t *)(p))[i] = (n); \ - } \ - } while (0) - -void tTSRowFree(SRow *pRow) { - if (pRow) { - taosMemoryFree(pRow); - } -} - -void tTSRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - uint8_t isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0; - STColumn *pTColumn = &pTSchema->columns[iCol]; - uint8_t flags = pRow->flags & (uint8_t)0xf; - SValue value; - - ASSERT(iCol < pTSchema->numOfCols); - ASSERT(flags); - ASSERT(pRow->sver == pTSchema->version); - - if (iCol == 0) { - value.ts = pRow->ts; - goto _return_value; - } - - if (flags == TSROW_HAS_NONE) { - goto _return_none; - } else if (flags == TSROW_HAS_NULL) { - goto _return_null; - } - - ASSERT(pRow->nData && pRow->pData); - - if (isTuple) { - uint8_t *pb = pRow->pData; - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t *p; - uint8_t b; - - // bit - switch (flags) { - case TSROW_HAS_VAL: - pf = pb; - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else { - goto _return_null; - } - case TSROW_HAS_VAL | TSROW_HAS_NONE: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else { - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_null; - } else { - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - b = GET_BIT2(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else if (b == 1) { - goto _return_null; - } else { - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - break; - } - default: - ASSERT(0); - } - - ASSERT(pf); - - p = pf + pTColumn->offset; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pv = pf + pTSchema->flen; - p = pv + *(VarDataOffsetT *)p; - } - tGetValue(p, &value, pTColumn->type); - goto _return_value; - } else { - STSKVRow *pRowK = (STSKVRow *)pRow->pData; - int16_t lidx = 0; - int16_t ridx = pRowK->nCols - 1; - uint8_t *p; - int16_t midx; - uint32_t n; - int16_t cid; - - ASSERT(pRowK->nCols > 0); - - if (pRow->flags & TSROW_KV_SMALL) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; - } else if (pRow->flags & TSROW_KV_MID) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; - } else if (pRow->flags & TSROW_KV_BIG) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; - } else { - ASSERT(0); - } - while (lidx <= ridx) { - midx = (lidx + ridx) / 2; - - if (pRow->flags & TSROW_KV_SMALL) { - n = ((uint8_t *)pRowK->idx)[midx]; - } else if (pRow->flags & TSROW_KV_MID) { - n = ((uint16_t *)pRowK->idx)[midx]; - } else { - n = ((uint32_t *)pRowK->idx)[midx]; - } - - n += tGetI16v(p + n, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - goto _return_null; - } else { - n += tGetValue(p + n, &value, pTColumn->type); - goto _return_value; - } - - return; - } else if (TABS(cid) > pTColumn->colId) { - ridx = midx - 1; - } else { - lidx = midx + 1; - } - } - - // not found, return NONE - goto _return_none; - } - -_return_none: - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - -_return_null: - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - -_return_value: - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); - return; -} - -int32_t tTSRowToArray(SRow *pRow, STSchema *pTSchema, SArray **ppArray) { - int32_t code = 0; - SColVal cv; - - (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); - if (*ppArray == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { - tTSRowGet(pRow, pTSchema, iColumn, &cv); - taosArrayPush(*ppArray, &cv); - } - - -_exit: - return code; -} -#endif - // STSchema ======================================== void tTSchemaDestroy(STSchema *pTSchema) { if (pTSchema) taosMemoryFree(pTSchema); From 75b2bb09c195ba1708f829e1ea74cd21def40930 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Nov 2022 14:58:57 +0800 Subject: [PATCH 057/165] more code --- source/common/src/tdataformat.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 44ececee58..dd64c20a71 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -662,8 +662,6 @@ SColVal *tRowIterNext(SRowIter *pIter) { case (HAS_NULL | HAS_NONE): bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); break; - case HAS_VALUE: - break; case (HAS_VALUE | HAS_NONE): bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); if (bv) bv++; From fec994e12444340e974bcf626e4d3d0beaedfd04 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Nov 2022 15:04:07 +0800 Subject: [PATCH 058/165] avoid double compress --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4fb00b1a6d..eaef8184b0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -820,7 +820,7 @@ void cliSend(SCliConn* pConn) { uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); } - if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { + if (pHead->comp == 0 && pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); } From 37d6b62e0228459205ffb646befeee8f1470bd32 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Nov 2022 15:14:59 +0800 Subject: [PATCH 059/165] more code --- include/common/tdataformat.h | 1 - source/common/src/tdataformat.c | 5 ----- 2 files changed, 6 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 179b46db0a..0ad2a1d305 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -94,7 +94,6 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); void tRowIterClose(SRowIter **ppIter); SColVal *tRowIterNext(SRowIter *pIter); -SColVal *tRowIterNextNotNone(SRowIter *pIter); // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index dd64c20a71..823b9182c2 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -708,11 +708,6 @@ _exit: return &pIter->cv; } -SColVal *tRowIterNextNotNone(SRowIter *pIter) { - // todo - return NULL; -} - // STSchema ======================================== void tTSchemaDestroy(STSchema *pTSchema) { if (pTSchema) taosMemoryFree(pTSchema); From 865332d960862fbfee3ee6074b254731027292fc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Nov 2022 15:27:07 +0800 Subject: [PATCH 060/165] avoid double compress --- source/libs/transport/src/transCli.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index eaef8184b0..5b8f059e04 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -791,6 +791,7 @@ void cliSend(SCliConn* pConn) { int msgLen = transMsgLenFromCont(pMsg->contLen); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; @@ -820,10 +821,15 @@ void cliSend(SCliConn* pConn) { uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); } - if (pHead->comp == 0 && pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { - msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead); - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + if (pHead->comp == 0) { + if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { + msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead); + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + } + } else { + msgLen = (int32_t)ntohl((uint32_t)(pHead->msgLen)); } + tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, msgLen); From 2da5a6106e7b16a92752d0c71543efbad16f3e9d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Nov 2022 15:31:20 +0800 Subject: [PATCH 061/165] refactor: do some internal refactor. --- source/libs/executor/src/cachescanoperator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 922ed05653..0c97ce3f30 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -26,7 +26,7 @@ #include "ttypes.h" static SSDataBlock* doScanCache(SOperatorInfo* pOperator); -static void destroyLastrowScanOperator(void* param); +static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo); @@ -97,14 +97,14 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyLastrowScanOperator, NULL); + createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, NULL); pOperator->cost.openCost = 0; return pOperator; _error: pTaskInfo->code = code; - destroyLastrowScanOperator(pInfo); + destroyCacheScanOperator(pInfo); taosMemoryFree(pOperator); return NULL; } @@ -234,7 +234,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } -void destroyLastrowScanOperator(void* param) { +void destroyCacheScanOperator(void* param) { SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pBufferredRes); From bcac60ae3510f6950288a8054a5b6f905f5f158d Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 22 Nov 2022 15:32:51 +0800 Subject: [PATCH 062/165] fix: init/free SBatchDeleteReq for tsma --- source/dnode/vnode/src/sma/smaTimeRange.c | 5 ++++- source/dnode/vnode/src/tq/tqSink.c | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index f5ba7b5014..062a678e0a 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -203,10 +203,13 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char goto _err; } - SBatchDeleteReq deleteReq; + SBatchDeleteReq deleteReq = {0}; SSubmitReq *pSubmitReq = tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); + + taosArrayDestroy(deleteReq.deleteReqs); + if (!pSubmitReq) { smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 27bfea0534..4eff52d4a2 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -25,6 +25,8 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl SColumnInfoData* pGidCol = taosArrayGet(pDataBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + + tqDebug("stream delete msg: row %d", totRow); for (int32_t row = 0; row < totRow; row++) { @@ -236,6 +238,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); if (pDataBlock->info.type == STREAM_DELETE_RESULT) { pDeleteReq->suid = suid; + pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); continue; } From d1249017dda6c02506f8c618bf552af22bfee3ac Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 22 Nov 2022 15:35:53 +0800 Subject: [PATCH 063/165] fix: init/free SBatchDeleteReq for tsma --- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 062a678e0a..a2c9484693 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -207,7 +207,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char SSubmitReq *pSubmitReq = tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); - + // TODO deleteReq taosArrayDestroy(deleteReq.deleteReqs); From da535cd04ce37c72c16ad3cec1bd393c8737634b Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 22 Nov 2022 15:38:21 +0800 Subject: [PATCH 064/165] fix: init/free SBatchDeleteReq for tsma --- source/dnode/vnode/src/tq/tqSink.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 4eff52d4a2..30cde5d475 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -25,8 +25,6 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl SColumnInfoData* pGidCol = taosArrayGet(pDataBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - - tqDebug("stream delete msg: row %d", totRow); for (int32_t row = 0; row < totRow; row++) { From ea591977bcd22ed2163bf53bb22e348cbbb3d5ca Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Tue, 22 Nov 2022 15:48:47 +0800 Subject: [PATCH 065/165] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 39 +++++++++--------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 90d27c3b8d..f8c5f970c5 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1978,15 +1978,8 @@ class TdSuperTable: def drop(self, dbc, skipCheck = False): dbName = self._dbName if self.exists(dbc) : # if myself exists - fullTableName = dbName + '.' + self._stName - - try: - dbc.execute("DROP TABLE {}".format(fullTableName)) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table - pass - + fullTableName = dbName + '.' + self._stName + dbc.execute("DROP TABLE {}".format(fullTableName)) else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) @@ -2004,16 +1997,9 @@ class TdSuperTable: fullTableName = dbName + '.' + self._stName if dbc.existsSuperTable(self._stName): - if dropIfExists: - try: - dbc.execute("DROP TABLE {}".format(fullTableName)) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table - pass - - pass - # dbc.execute("DROP TABLE {}".format(fullTableName)) + if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -2049,13 +2035,12 @@ class TdSuperTable: except TmqError as e : pass - # consumer work only 30 sec + # consumer with random work life time_start = time.time() while 1: res = consumer.poll(1000) - if time.time() - time_start >5 : + if time.time() - time_start >random.randint(5,50) : break - # time.sleep(10) try: consumer.unsubscribe() except TmqError as e : @@ -2435,11 +2420,15 @@ class TaskDropSuperTable(StateTransitionTask): for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) try: - self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - pass + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) + if (errno2 in [0x362]): # mnode invalid table name + isSuccess = False + Logging.debug("[DB] Acceptable error when dropping a table") + continue # try to delete next regular table if (not tickOutput): @@ -2957,6 +2946,8 @@ class ThreadStacks: # stack info for all threads print(" {}".format(frame.line)) stackFrame += 1 print("-----> End of Thread Info ----->\n") + if self.current_time-last_sql_commit_time >100: # dead lock occured + print("maybe dead locked of thread {} ".format(shortTid)) class ClientManager: def __init__(self): From 7a81c2e2d0de73a45eff5e0a79204d6ae51c7227 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Tue, 22 Nov 2022 15:49:18 +0800 Subject: [PATCH 066/165] Update db.py From a6ac48700941b1084f386ac034b9f47ec8f1ccf5 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Tue, 22 Nov 2022 15:50:32 +0800 Subject: [PATCH 067/165] Update crash_gen_main.py From a8e65a6a2b1b92c6048fbe9f9190af6d1b557fdd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 15:56:06 +0800 Subject: [PATCH 068/165] test: add asan case --- tests/parallel_test/cases.task | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e4c6a40be8..25926e7da7 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -216,7 +216,7 @@ ,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim -,,n,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim @@ -227,7 +227,7 @@ ,,y,script,./test.sh -f tsim/stream/triggerSession0.sim ,,y,script,./test.sh -f tsim/stream/partitionby.sim ,,y,script,./test.sh -f tsim/stream/partitionby1.sim -,,n,script,./test.sh -f tsim/stream/schedSnode.sim +,,y,script,./test.sh -f tsim/stream/schedSnode.sim ,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim @@ -800,7 +800,7 @@ ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 @@ -813,7 +813,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 @@ -860,8 +860,8 @@ ,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 @@ -874,7 +874,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 @@ -893,11 +893,11 @@ ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 @@ -919,7 +919,7 @@ ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 @@ -980,14 +980,14 @@ ,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 @@ -995,7 +995,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 From ca3c1f714749851f99f26e8d4f6483c57f30a041 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Tue, 22 Nov 2022 16:08:14 +0800 Subject: [PATCH 069/165] add docker run exec script --- tests/pytest/docker_exec_service.sh | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/pytest/docker_exec_service.sh diff --git a/tests/pytest/docker_exec_service.sh b/tests/pytest/docker_exec_service.sh new file mode 100644 index 0000000000..4156a0bae5 --- /dev/null +++ b/tests/pytest/docker_exec_service.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +mkdir -p /data/wz/crash_gen_logs/ +logdir='/data/wz/crash_gen_logs/' +date_tag=`date +%Y%m%d-%H%M%S` +hostname='vm_valgrind_' + +for i in {1..50} +do + echo $i + # create docker and start crash_gen + log_dir=${logdir}${hostname}${date_tag}_${i} + docker run -d --hostname=${hostname}${date_tag}_${i} --name ${hostname}${date_tag}_${i} --privileged -v ${log_dir}:/corefile/ -- crash_gen:v1.0 sleep 99999999999999 + echo create docker ${hostname}${date_tag}_${i} + docker exec -d ${hostname}${date_tag}_${i} sh -c 'rm -rf /home/taos-connector-python' + docker cp /data/wz/TDengine ${hostname}${date_tag}_${i}:/home/TDengine + docker cp /data/wz/taos-connector-python ${hostname}${date_tag}_${i}:/home/taos-connector-python + echo copy TDengine in container done! + docker exec ${hostname}${date_tag}_${i} sh -c 'sh /home/TDengine/tests/pytest/auto_run_valgrind.sh ' + if [ $? -eq 0 ] + then + echo crash_gen exit as expect , run success + + # # clear docker which success + docker stop ${hostname}${date_tag}_${i} + docker rm -f ${hostname}${date_tag}_${i} + else + docker stop ${hostname}${date_tag}_${i} + echo crash_gen exit error , run failed + fi +done \ No newline at end of file From 0937a2f6b928a408f460716934e8263aad1fb96c Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Tue, 22 Nov 2022 16:23:49 +0800 Subject: [PATCH 070/165] docs: add TDengine Knowledge Map --- docs/en/05-get-started/index.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index d80ec02268..e42c9b5a41 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -20,6 +20,17 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; ``` +## Study TDengine Knowledge Map + +The TDengine Knowledge Map covers the various knowledge points of TDengine, revealing the invocation relationships and data flow between various conceptual entities. Learning and understanding the TDengine Knowledge Map will help you quickly master the TDengine knowledge system. + +

+
+ +
Diagram 1. TDengine Knowledge Map
+
+
+ ### Join TDengine Community From 2020ea0a7683d96bedb6361d44737518707abcaf Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sun, 20 Nov 2022 10:15:26 +0800 Subject: [PATCH 071/165] fix:stream fill crash --- source/common/src/tdatablock.c | 1 + source/libs/executor/inc/executorimpl.h | 17 ++++++++++++++ source/libs/executor/inc/tfill.h | 16 ------------- source/libs/executor/src/executorimpl.c | 2 ++ source/libs/executor/src/tfill.c | 31 +++++++++++-------------- 5 files changed, 34 insertions(+), 33 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c2a5377e3..c2cdfc056a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1962,6 +1962,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) memset(pBuf, 0, sizeof(pBuf)); char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); + dataSize = TMIN(dataSize, 50); memcpy(pBuf, varDataVal(pData), dataSize); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) return dumpBuf; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f179c7bd41..7dc9e9ab03 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -754,6 +754,23 @@ typedef struct SStreamPartitionOperatorInfo { SSDataBlock* pDelRes; } SStreamPartitionOperatorInfo; +typedef struct SStreamFillSupporter { + int32_t type; // fill type + SInterval interval; + SResultRowData prev; + SResultRowData cur; + SResultRowData next; + SResultRowData nextNext; + SFillColInfo* pAllColInfo; // fill exprs and not fill exprs + SExprSupp notFillExprSup; + int32_t numOfAllCols; // number of all exprs, including the tags columns + int32_t numOfFillCols; + int32_t numOfNotFillCols; + int32_t rowSize; + SSHashObj* pResMap; + bool hasDelete; +} SStreamFillSupporter; + typedef struct SStreamFillOperatorInfo { SStreamFillSupporter* pFillSup; SSDataBlock* pRes; diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 2d8df81dbd..b0017fef50 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -111,22 +111,6 @@ typedef struct SStreamFillInfo { int32_t delIndex; } SStreamFillInfo; -typedef struct SStreamFillSupporter { - int32_t type; // fill type - SInterval interval; - SResultRowData prev; - SResultRowData cur; - SResultRowData next; - SResultRowData nextNext; - SFillColInfo* pAllColInfo; // fill exprs and not fill exprs - int32_t numOfAllCols; // number of all exprs, including the tags columns - int32_t numOfFillCols; - int32_t numOfNotFillCols; - int32_t rowSize; - SSHashObj* pResMap; - bool hasDelete; -} SStreamFillSupporter; - int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7e955100c..20c39ec921 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2056,6 +2056,8 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_COLUMN) { taosMemoryFreeClear(pExprInfo->base.pParam[j].pCol); + } else if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_VALUE) { + taosVariantDestroy(&pExprInfo->base.pParam[j].param); } } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index c41376b2dc..9908f35818 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -709,6 +709,7 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pResMap = NULL; releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); pFillSup->cur.pRowVal = NULL; + cleanupExprSupp(&pFillSup->notFillExprSup); taosMemoryFree(pFillSup); return NULL; @@ -1417,25 +1418,13 @@ static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + + pDstBlock->info.rows = 0; + pSup = &pInfo->pFillSup->notFillExprSup; + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); pDstBlock->info.groupId = pSrcBlock->info.groupId; - SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, pInfo->primaryTsCol); - SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, pInfo->primarySrcSlotId); - colDataAssign(pDst, pSrc, pDstBlock->info.rows, &pDstBlock->info); - - int32_t numOfNotFill = pInfo->pFillSup->numOfAllCols - pInfo->pFillSup->numOfFillCols; - for (int32_t i = 0; i < numOfNotFill; ++i) { - SFillColInfo* pCol = &pInfo->pFillSup->pAllColInfo[i + pInfo->pFillSup->numOfFillCols]; - ASSERT(pCol->notFillCol); - - SExprInfo* pExpr = pCol->pExpr; - int32_t srcSlotId = pExpr->base.pParam[0].pCol->slotId; - int32_t dstSlotId = pExpr->base.resSchema.slotId; - - SColumnInfoData* pDst1 = taosArrayGet(pDstBlock->pDataBlock, dstSlotId); - SColumnInfoData* pSrc1 = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); - colDataAssign(pDst1, pSrc1, pDstBlock->info.rows, &pDstBlock->info); - } blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); } @@ -1577,6 +1566,14 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod destroyStreamFillSupporter(pFillSup); return NULL; } + + SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pFillSup->pResMap = tSimpleHashInit(16, hashFn); pFillSup->hasDelete = false; From ac0ba146b320cc3c6003c7d12f6106be99ac9445 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Tue, 22 Nov 2022 16:33:10 +0800 Subject: [PATCH 072/165] fix/release add taosx --- packaging/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index c07331a0df..a3334e734d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -221,12 +221,12 @@ if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" = # community-version compile cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} elif [ "$verMode" == "cloud" ]; then - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} + cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} elif [ "$verMode" == "cluster" ]; then if [[ "$dbName" != "taos" ]]; then replace_enterprise_$dbName fi - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} + cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} fi else echo "input cpuType=${cpuType} error!!!" From 6f5d90658ef0ccfad5c0aeefe02369539581d76d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 22 Nov 2022 16:40:18 +0800 Subject: [PATCH 073/165] fix(wal): add debug log --- source/libs/wal/src/walWrite.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index caae669e4a..30bdc4eddc 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -324,12 +324,17 @@ int32_t walEndSnapshot(SWal *pWal) { // find files safe to delete SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); if (pInfo) { + SWalFileInfo *pLastFileInfo = taosArrayGetLast(pWal->fileInfoSet); + wDebug("vgId:%d, wal search found file info: first:%" PRId64 "last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, + pInfo->lastVer); if (ver >= pInfo->lastVer) { - //pInfo--; pInfo++; + wDebug("vgId:%d, wal remove advance one file: first:%" PRId64 "last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, + pInfo->lastVer); } - if (POINTER_DISTANCE(pInfo, pWal->fileInfoSet->pData) > 0) { - wDebug("vgId:%d, wal end remove for %" PRId64, pWal->cfg.vgId, pInfo->firstVer); + if (pInfo <= pLastFileInfo) { + wDebug("vgId:%d, wal end remove for first:%" PRId64 "last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, + pInfo->lastVer); } else { wDebug("vgId:%d, wal no remove", pWal->cfg.vgId); } From 42e6f59e49ec6bd9869ac15919dec9efe367c1d1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Nov 2022 16:53:58 +0800 Subject: [PATCH 074/165] avoid invalid vgroup id --- source/libs/transport/src/transSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 8dd3628c5f..f093d84db6 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -398,7 +398,7 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { pHead->magicNum = htonl(TRANS_MAGIC_NUM); // handle invalid drop_task resp, TD-20098 - if (pMsg->msgType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { transQueuePop(&pConn->srvMsgs); destroySmsg(smsg); return -1; From ee18f991aa3b108bb23e19df4401a2f6406533e3 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 22 Nov 2022 16:54:42 +0800 Subject: [PATCH 075/165] test: add test case for tmq --- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmqDnodeRestart1.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e4c6a40be8..835ca61fd6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -704,6 +704,7 @@ ,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 714f10b362..982cc0a631 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -149,6 +149,8 @@ class TDTestCase: tmqCom.waitSubscriptionExit(tdSql, topicFromStb) tdSql.query("drop topic %s"%topicFromStb) + + tmqCom.stopTmqSimProcess(processorName="tmq_sim") tdLog.printNoPrefix("======== test case 1 end ...... ") @@ -178,6 +180,8 @@ class TDTestCase: paraDict['vgroups'] = self.vgroups paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() tdLog.info("create topics from stb") topicFromDb = 'topic_db' @@ -203,10 +207,10 @@ class TDTestCase: tmqCom.getStartCommitNotifyFromTmqsim('cdb',1) tdLog.info("create some new child table and insert data for latest mode") - paraDict["batchNum"] = 100 + paraDict["batchNum"] = 10 paraDict["ctbPrefix"] = 'newCtb' - paraDict["ctbNum"] = 10 - paraDict["rowsPerTbl"] = 10 + paraDict["ctbNum"] = 100 + paraDict["rowsPerTbl"] = 100 tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) tdLog.info("================= restart dnode ===========================") From 067a568d85e0487212c21f436d281cb8b07456ae Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Tue, 22 Nov 2022 17:01:46 +0800 Subject: [PATCH 076/165] docs: fix heading level --- docs/en/05-get-started/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index e42c9b5a41..251581e98f 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -31,7 +31,7 @@ The TDengine Knowledge Map covers the various knowledge points of TDengine, reve -### Join TDengine Community +## Join TDengine Community
From 5b8d3d57bb2caa718cc61477e596d6b34711474d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 22 Nov 2022 17:01:59 +0800 Subject: [PATCH 077/165] fix:Direct leak in timewindowoperator.c --- source/libs/executor/src/timewindowoperator.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 1ec7d6d26b..dd02ce9cd4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1618,6 +1618,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { nodesDestroyNode((SNode*)pInfo->pPhyNode); colDataDestroy(&pInfo->twAggSup.timeWindowData); cleanupGroupResInfo(&pInfo->groupResInfo); + cleanupExprSupp(&pInfo->scalarSupp); taosMemoryFreeClear(param); } @@ -5363,15 +5364,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); - if (pIntervalPhyNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { From 13446718697bb51ac11ba31f3a2ef08187fd6bef Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 22 Nov 2022 17:07:39 +0800 Subject: [PATCH 078/165] fix(wal): add debug log --- source/libs/wal/src/walWrite.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 30bdc4eddc..c5c8173f63 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -325,28 +325,35 @@ int32_t walEndSnapshot(SWal *pWal) { SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); if (pInfo) { SWalFileInfo *pLastFileInfo = taosArrayGetLast(pWal->fileInfoSet); - wDebug("vgId:%d, wal search found file info: first:%" PRId64 "last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, + wDebug("vgId:%d, wal search found file info: first:%" PRId64 " last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, pInfo->lastVer); if (ver >= pInfo->lastVer) { pInfo++; - wDebug("vgId:%d, wal remove advance one file: first:%" PRId64 "last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, + wDebug("vgId:%d, wal remove advance one file: first:%" PRId64 " last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, pInfo->lastVer); } if (pInfo <= pLastFileInfo) { - wDebug("vgId:%d, wal end remove for first:%" PRId64 "last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, + wDebug("vgId:%d, wal end remove for first:%" PRId64 " last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer, pInfo->lastVer); } else { wDebug("vgId:%d, wal no remove", pWal->cfg.vgId); } // iterate files, until the searched result for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if ((pWal->cfg.retentionSize != -1 && newTotSize > pWal->cfg.retentionSize) || - (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) { + wDebug("vgId:%d, wal check remove file %" PRId64 "(file size %" PRId64 " close ts %" PRId64 + "), new tot size %" PRId64, + pWal->cfg.vgId, iter->firstVer, iter->fileSize, iter->closeTs, newTotSize); + if (((pWal->cfg.retentionSize == 0) || (pWal->cfg.retentionSize != -1 && newTotSize > pWal->cfg.retentionSize)) || + ((pWal->cfg.retentionPeriod == 0) || + (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts))) { // delete according to file size or close time + wDebug("vgId:%d, check pass", pWal->cfg.vgId); deleteCnt++; newTotSize -= iter->fileSize; } + wDebug("vgId:%d, check not pass", pWal->cfg.vgId); } + wDebug("vgId:%d, wal should delete %d files", pWal->cfg.vgId, deleteCnt); int32_t actualDelete = 0; char fnameStr[WAL_FILE_LEN]; // remove file From 2c37b1eb6537e4b3bb43b6bfbad5d1361f2637b7 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 22 Nov 2022 17:30:32 +0800 Subject: [PATCH 079/165] Update 06-select.md --- docs/zh/12-taos-sql/06-select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 9d4faae23a..3b681f401c 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -302,7 +302,7 @@ SELECT TIMEZONE(); ### 语法 ```txt -WHERE (column|tbname) **match/MATCH/nmatch/NMATCH** _regex_ +WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_ ``` ### 正则表达式规范 From 7715dab468c61602acefca092b4cfa64f2d9561a Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 22 Nov 2022 17:31:07 +0800 Subject: [PATCH 080/165] Update 06-select.md --- docs/en/12-taos-sql/06-select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 0c55578efa..c087a9e9fb 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -301,7 +301,7 @@ SELECT TIMEZONE(); ### Syntax ```txt -WHERE (column|tbname) **match/MATCH/nmatch/NMATCH** _regex_ +WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_ ``` ### Specification From 7c7b2f1cef5539dc2d8f58bfba9189f0ed98cbaa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Nov 2022 18:06:01 +0800 Subject: [PATCH 081/165] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 15 ++++++++++----- source/dnode/vnode/src/tsdb/tsdbRead.c | 10 +++++----- source/libs/executor/src/cachescanoperator.c | 11 ++++++++--- source/libs/function/src/builtinsimpl.c | 4 +++- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 2ae3115c0a..b87e5d5503 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -287,12 +287,17 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 hasRes = true; p->ts = pColVal->ts; - uint8_t* px = p->colVal.value.pData; - p->colVal = pColVal->colVal; + if (!IS_VAR_DATA_TYPE(pColVal->colVal.type)) { + p->colVal = pColVal->colVal; + } else { + if (COL_VAL_IS_VALUE(&pColVal->colVal)) { + memcpy(p->colVal.value.pData, pColVal->colVal.value.pData, pColVal->colVal.value.nData); + } - if (COL_VAL_IS_VALUE(&pColVal->colVal) && IS_VAR_DATA_TYPE(pColVal->colVal.type)) { - p->colVal.value.pData = px; - memcpy(px, pColVal->colVal.value.pData, pColVal->colVal.value.nData); + p->colVal.value.nData = pColVal->colVal.value.nData; + p->colVal.type = pColVal->colVal.type; + p->colVal.flag = pColVal->colVal.flag; + p->colVal.cid = pColVal->colVal.cid; } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 69fae4d481..117729b3bd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -305,7 +305,7 @@ static void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableKeyInfo* idList, int32_t numOfTables) { +static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SHashObj* pTableMap = @@ -315,10 +315,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } int64_t st = taosGetTimestampUs(); - initBlockScanInfoBuf(&pTsdbReader->blockInfoBuf, numOfTables); + initBlockScanInfoBuf(pBuf, numOfTables); for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(&pTsdbReader->blockInfoBuf, j); + STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); pScanInfo->uid = idList[j].uid; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { int64_t skey = pTsdbReader->window.skey; @@ -3785,9 +3785,9 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } STsdbReader* p = (pReader->innerReader[0] != NULL)? pReader->innerReader[0]:pReader; - pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList, numOfTables); + pReader->status.pTableMap = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, numOfTables); if (pReader->status.pTableMap == NULL) { - tsdbReaderClose(pReader); + tsdbReaderClose(p); *ppReader = NULL; code = TSDB_CODE_TDB_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 0c97ce3f30..5e5c01201f 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -157,9 +157,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId); SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId); - char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); - bool isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes); - colDataAppend(pDst, 0, p, isNull); + if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) { + colDataAppendNULL(pDst, 0); + } else { + char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); + colDataAppend(pDst, 0, p, false); + } } pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); @@ -226,6 +229,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); return pInfo->pRes; + } else { + pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); } } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 87e15370e4..4d69b4d45c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6568,7 +6568,9 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { numOfElems++; - char* data = colDataGetData(pInputCol, i); + bool isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL); + char* data = isNull ? NULL : colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); From f2e2aa77227d57b3655e6f82b80c7b5906152bc7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 22 Nov 2022 18:18:31 +0800 Subject: [PATCH 082/165] fix: insert stable error --- source/libs/parser/src/parInsertSql.c | 33 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 411adc680c..61951b171b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -792,6 +792,8 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt *pMissCache = true; } else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) { code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + } else if (!isStb && TSDB_SUPER_TABLE == (*pTableMeta)->tableType) { + code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } } return code; @@ -1571,16 +1573,16 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } -static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode** pOutput) { +static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (NULL == pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } - if (pCxt->pStmtCb) { + if (pCxt->pComCxt->pStmtCb) { TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } - pStmt->pSql = pCxt->pSql; + pStmt->pSql = pCxt->pComCxt->pSql; pStmt->freeHashFunc = insDestroyBlockHashmap; pStmt->freeArrayFunc = insDestroyBlockArrayList; @@ -1604,7 +1606,7 @@ static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode** return TSDB_CODE_SUCCESS; } -static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) { +static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) { SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1667,11 +1669,15 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpSt sizeof(SVgroupInfo)); } -static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt, bool isStb) { +static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, + SVnodeModifOpStmt* pStmt, bool isStb) { int32_t code = checkAuthFromMetaData(pMetaData->pUser); if (TSDB_CODE_SUCCESS == code) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); } + if (TSDB_CODE_SUCCESS == code && !isStb && TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { + code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); + } if (TSDB_CODE_SUCCESS == code) { code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } @@ -1696,24 +1702,25 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { pCatalogReq->pUser = NULL; } -static int32_t setVnodeModifOpStmt(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, +static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt) { clearCatalogReq(pCatalogReq); if (pStmt->usingTableProcessing) { - return getTableSchemaFromMetaData(pMetaData, pStmt, true); + return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); } - return getTableSchemaFromMetaData(pMetaData, pStmt, false); + return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } -static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) { +static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) { nodesDestroyNode(pQuery->pRoot); int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; - (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); + (*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, + &pStmt->pTableBlockHashObj); if (NULL == pStmt->pVgroupsHashObj) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } @@ -1729,13 +1736,13 @@ static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) { return code; } -static int32_t initInsertQuery(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, +static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SQuery** pQuery) { if (NULL == *pQuery) { return createInsertQuery(pCxt, pQuery); } - if (NULL != pCxt->pStmtCb) { + if (NULL != pCxt->pComCxt->pStmtCb) { return resetVnodeModifOpStmt(pCxt, *pQuery); } @@ -1896,7 +1903,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .usingDuplicateTable = false, }; - int32_t code = initInsertQuery(pCxt, pCatalogReq, pMetaData, pQuery); + int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); } From b6b00cf3a4ced5b23ac9c211b5aa435e17c9ca2d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Nov 2022 18:30:57 +0800 Subject: [PATCH 083/165] fix: remove double free one-liners --- source/dnode/vnode/src/tq/tqSink.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b2624d1bc1..f54ed3fc63 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -349,7 +349,6 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d .contLen = len + sizeof(SMsgHead), }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(serializedDeleteReq); tqDebug("failed to put delete req into write-queue since %s", terrstr()); } } else { @@ -541,7 +540,6 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(ret); tqDebug("failed to put into write-queue since %s", terrstr()); } } From 5922c725b1a7d589c4f78afbadd4d98c806e2309 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 22 Nov 2022 20:55:24 +0800 Subject: [PATCH 084/165] fix: insert stable error --- source/libs/parser/src/parInsertSql.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 61951b171b..9c16bf645f 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -227,6 +227,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, S } } + if (pColList->cols[0].valStat == VAL_STAT_NONE) { + return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + } + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; if (!isOrdered) { From f34a5c40996ff76cce51b43c6ecc5819921bf095 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 21:24:21 +0800 Subject: [PATCH 085/165] test: add asan case --- tests/parallel_test/cases.task | 10 +++++----- tests/script/sh/checkAsan.sh | 13 +++++++++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 25926e7da7..e08cce6f9b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -465,8 +465,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -R -,,,system-test,python3 ./test.py -f 2-query/concat.py -,,,system-test,python3 ./test.py -f 2-query/concat.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -R ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py @@ -739,7 +739,7 @@ ,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2 @@ -833,7 +833,7 @@ ,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 3 @@ -926,7 +926,7 @@ ,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 4 diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 074956534f..8b478384cf 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -21,15 +21,24 @@ LOG_DIR=$TAOS_DIR/sim/asan error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` +python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # ignore + +# TD-20368 +# /root/TDengine/contrib/zlib/trees.c:873:5: runtime error: null pointer passed as argument 2, which is declared to never be null + +# TD-20494 TD-20452 # /root/TDengine/source/libs/scalar/src/sclfunc.c:735:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'signed char' # /root/TDengine/source/libs/scalar/src/sclfunc.c:790:11: runtime error: 3.4e+38 is outside the range of representable values of type 'long int' # /root/TDengine/source/libs/scalar/src/sclfunc.c:772:11: runtime error: 3.52344e+09 is outside the range of representable values of type 'int' # /root/TDengine/source/libs/scalar/src/sclfunc.c:753:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'short int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | wc -l` -python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` +# TD-20569 +# /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int' +# /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' + +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" From 94cba845aff3e23e707e37619f4ff0c052af2b0f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 21:44:34 +0800 Subject: [PATCH 086/165] test: add asan case --- tests/parallel_test/cases.task | 430 ++++++++++++++++----------------- 1 file changed, 215 insertions(+), 215 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e08cce6f9b..5413e09bc6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -451,8 +451,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -R -,,,system-test,python3 ./test.py -f 2-query/avg.py -,,,system-test,python3 ./test.py -f 2-query/avg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py @@ -469,8 +469,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -R ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R +,,y,system-test,./pytest.sh python3 -f 2-query/concat_ws2.py +,,y,system-test,./pytest.sh python3 -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py @@ -515,14 +515,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -R -,,,system-test,python3 ./test.py -f 2-query/interp.py -,,,system-test,python3 ./test.py -f 2-query/interp.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -R -,,,system-test,python3 ./test.py -f 2-query/last_row.py -,,,system-test,python3 ./test.py -f 2-query/last_row.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py @@ -531,10 +531,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R -,,,system-test,python3 ./test.py -f 2-query/lower.py -,,,system-test,python3 ./test.py -f 2-query/lower.py -R -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py @@ -555,8 +555,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -R -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py @@ -573,12 +573,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -R -,,,system-test,python3 ./test.py -f 2-query/substr.py -,,,system-test,python3 ./test.py -f 2-query/substr.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py -R -,,,system-test,python3 ./test.py -f 2-query/tail.py -,,,system-test,python3 ./test.py -f 2-query/tail.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py @@ -603,8 +603,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -R -,,,system-test,python3 ./test.py -f 2-query/unique.py -,,,system-test,python3 ./test.py -f 2-query/unique.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py @@ -724,11 +724,11 @@ ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 2 @@ -736,13 +736,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 2 @@ -781,230 +781,230 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 #develop test From d4c739414300a31e681fe236974ad5bcc47f01f5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 22 Nov 2022 22:03:08 +0800 Subject: [PATCH 087/165] fix(stream): memory leak --- source/dnode/vnode/src/tq/tqPush.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 12d5b4112b..f89bc20362 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -308,9 +308,8 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) } if (vnodeIsRoleLeader(pTq->pVnode)) { + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; - void* data = taosMemoryMalloc(msgLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; From 272a161a610caeddab6338c746c218084ed89547 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 22 Nov 2022 22:06:39 +0800 Subject: [PATCH 088/165] fix: build data from mem/imem first when large nchar column incur fileBlockShouldLoad since SDataBlockToLoadInfo.moreThanCapacity --- source/dnode/vnode/src/tsdb/tsdbRead.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bb2de2ec93..d4f87525a5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2805,6 +2805,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); + } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + // data in memory that are earlier than current file block + // rows in buffer should be less than the file block in asc, greater than file block in desc + // process mem/imem data first even if fileBlockShouldLoad + int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { @@ -2813,11 +2819,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - // data in memory that are earlier than current file block - // rows in buffer should be less than the file block in asc, greater than file block in desc - int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; - code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { // only return the rows in last block From 0d60f24db59ddcbb2f9d0701ad08c15d127361db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Nov 2022 22:42:44 +0800 Subject: [PATCH 089/165] refactor: do some internal refactor. --- source/common/src/tdatablock.c | 2 - source/libs/executor/CMakeLists.txt | 4 - source/libs/executor/inc/executorimpl.h | 146 +- source/libs/executor/src/cachescanoperator.c | 20 +- source/libs/executor/src/exchangeoperator.c | 88 + source/libs/executor/src/executorimpl.c | 192 -- source/libs/executor/src/groupoperator.c | 30 + source/libs/executor/src/joinoperator.c | 15 + source/libs/executor/src/projectoperator.c | 18 + source/libs/executor/src/scanoperator.c | 1710 +---------------- source/libs/executor/src/sortoperator.c | 12 + source/libs/executor/src/sysscanoperator.c | 1774 ++++++++++++++++++ 12 files changed, 1964 insertions(+), 2047 deletions(-) create mode 100644 source/libs/executor/src/sysscanoperator.c diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c2a5377e3..b5fffea73d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1084,8 +1084,6 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { return 0; } -int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; } - int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { // Allocate the additional buffer. int64_t p0 = taosGetTimestampUs(); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 89d08b3078..8b3d04e32c 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -2,10 +2,6 @@ aux_source_directory(src EXECUTOR_SRC) #add_library(executor ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC}) -#set_target_properties(executor PROPERTIES -# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" -# INTERFACE_INCLUDE_DIRECTORIES "${TD_SOURCE_DIR}/include/libs/executor" -# ) target_link_libraries(executor PRIVATE os util common function parser planner qcom vnode scalar nodes index stream diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f179c7bd41..f85bddcc31 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -235,16 +235,6 @@ typedef enum { #define COL_MATCH_FROM_COL_ID 0x1 #define COL_MATCH_FROM_SLOT_ID 0x2 -typedef struct SSourceDataInfo { - int32_t index; - SRetrieveTableRsp* pRsp; - uint64_t totalRows; - int64_t startTime; - int32_t code; - EX_SOURCE_STATUS status; - const char* taskId; -} SSourceDataInfo; - typedef struct SLoadRemoteDataInfo { uint64_t totalSize; // total load bytes from remote uint64_t totalRows; // total number of rows @@ -371,23 +361,8 @@ typedef struct STagScanInfo { SColMatchInfo matchInfo; int32_t curPos; SReadHandle readHandle; - STableListInfo* pTableList; } STagScanInfo; -typedef struct SLastrowScanInfo { - SSDataBlock* pRes; - SReadHandle readHandle; - void* pLastrowReader; - SColMatchInfo matchInfo; - int32_t* pSlotIds; - SExprSupp pseudoExprSup; - int32_t retrieveType; - int32_t currentGroupIndex; - SSDataBlock* pBufferredRes; - SArray* pUidList; - int32_t indexOfBufferedRes; -} SLastrowScanInfo; - typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, @@ -504,40 +479,6 @@ typedef struct { SSnapContext* sContext; } SStreamRawScanInfo; -typedef struct SSysTableIndex { - int8_t init; - SArray* uids; - int32_t lastIdx; -} SSysTableIndex; - -typedef struct SSysTableScanInfo { - SRetrieveMetaTableRsp* pRsp; - SRetrieveTableReq req; - SEpSet epSet; - tsem_t ready; - SReadHandle readHandle; - int32_t accountId; - const char* pUser; - bool sysInfo; - bool showRewrite; - SNode* pCondition; // db_name filter condition, to discard data that are not in current database - SMTbCursor* pCur; // cursor for iterate the local table meta store. - SSysTableIndex* pIdx; // idx for local table meta - SColMatchInfo matchInfo; - SName name; - SSDataBlock* pRes; - int64_t numOfBlocks; // extract basic running information. - SLoadRemoteDataInfo loadInfo; -} SSysTableScanInfo; - -typedef struct SBlockDistInfo { - SSDataBlock* pResBlock; - STsdbReader* pHandle; - SReadHandle readHandle; - uint64_t uid; // table uid -} SBlockDistInfo; - -// todo remove this typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; @@ -603,24 +544,6 @@ typedef struct SAggOperatorInfo { SExprSupp scalarExprSup; } SAggOperatorInfo; -typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SLimitInfo limitInfo; - bool mergeDataBlocks; - SSDataBlock* pFinalRes; -} SProjectOperatorInfo; - -typedef struct SIndefOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SExprSupp scalarSup; - uint64_t groupId; - SSDataBlock* pNextGroupRes; -} SIndefOperatorInfo; - typedef struct SFillOperatorInfo { struct SFillInfo* pFillInfo; SSDataBlock* pRes; @@ -638,42 +561,12 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -typedef struct SGroupbyOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - bool isInit; // denote if current val is initialized or not - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SGroupResInfo groupResInfo; - SExprSupp scalarSup; -} SGroupbyOperatorInfo; - typedef struct SDataGroupInfo { uint64_t groupId; int64_t numOfRows; SArray* pPageList; } SDataGroupInfo; -// The sort in partition may be needed later. -typedef struct SPartitionOperatorInfo { - SOptrBasicInfo binfo; - SArray* pGroupCols; - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SHashObj* pGroupSet; // quick locate the window object for each result - - SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file - int32_t rowCapacity; // maximum number of rows for each buffer page - int32_t* columnOffset; // start position for each column data - SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - int32_t groupIndex; // group index - int32_t pageIndex; // page index of current group - SExprSupp scalarSup; -} SPartitionOperatorInfo; - typedef struct SWindowRowsSup { STimeWindow win; TSKEY prevTs; @@ -800,33 +693,6 @@ typedef struct SStateWindowOperatorInfo { STimeWindowAggSupp twAggSup; } SStateWindowOperatorInfo; -typedef struct SSortOperatorInfo { - SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SColMatchInfo matchInfo; - int32_t bufPageSize; - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SLimitInfo limitInfo; -} SSortOperatorInfo; - -typedef struct SJoinOperatorInfo { - SSDataBlock* pRes; - int32_t joinType; - int32_t inputOrder; - - SSDataBlock* pLeft; - int32_t leftPos; - SColumnInfo leftCol; - - SSDataBlock* pRight; - int32_t rightPos; - SColumnInfo rightCol; - SNode* pCondAfterMerge; -} SJoinOperatorInfo; - #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) @@ -850,7 +716,6 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator); @@ -880,9 +745,6 @@ void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); -int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); -int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); - SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); @@ -965,9 +827,8 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -void setTaskKilled(SExecTaskInfo* pTaskInfo); -void queryCostStatis(SExecTaskInfo* pTaskInfo); - +void setTaskKilled(SExecTaskInfo* pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); void destroyOperatorInfo(SOperatorInfo* pOperator); int32_t getMaximumIdleDurationSec(); @@ -995,9 +856,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); -int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); - STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 5e5c01201f..6b5f773fe3 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -25,6 +25,20 @@ #include "thash.h" #include "ttypes.h" +typedef struct SCacheRowsScanInfo { + SSDataBlock* pRes; + SReadHandle readHandle; + void* pLastrowReader; + SColMatchInfo matchInfo; + int32_t* pSlotIds; + SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock* pBufferredRes; + SArray* pUidList; + int32_t indexOfBufferedRes; +} SCacheRowsScanInfo; + static SSDataBlock* doScanCache(SOperatorInfo* pOperator); static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); @@ -33,7 +47,7 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; - SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo)); + SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -114,7 +128,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { return NULL; } - SLastrowScanInfo* pInfo = pOperator->info; + SCacheRowsScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableListInfo* pTableList = pTaskInfo->pTableInfoList; @@ -240,7 +254,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } void destroyCacheScanOperator(void* param) { - SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; + SCacheRowsScanInfo* pInfo = (SCacheRowsScanInfo*)param; blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pBufferredRes); taosMemoryFree(pInfo->pSlotIds); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index a28066003a..c858536bb1 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -41,6 +41,16 @@ typedef struct SFetchRspHandleWrapper { int32_t sourceIndex; } SFetchRspHandleWrapper; +typedef struct SSourceDataInfo { + int32_t index; + SRetrieveTableRsp* pRsp; + uint64_t totalRows; + int64_t startTime; + int32_t code; + EX_SOURCE_STATUS status; + const char* taskId; +} SSourceDataInfo; + static void destroyExchangeOperatorInfo(void* param); static void freeBlock(void* pParam); static void freeSourceDataInfo(void* param); @@ -52,6 +62,7 @@ static int32_t getCompletedSources(const SArray* pArray); static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); +static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo) { @@ -647,3 +658,80 @@ int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; return TSDB_CODE_SUCCESS; } + +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { + if (pLimitInfo->remainGroupOffset > 0) { + if (pLimitInfo->currentGroupId == 0) { // it is the first group + pLimitInfo->currentGroupId = pBlock->info.groupId; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; + + // ignore data block in current group + if (pLimitInfo->remainGroupOffset > 0) { + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } + } + + // set current group id of the project operator + pLimitInfo->currentGroupId = pBlock->info.groupId; + } + + // here check for a new group data, we need to handle the data of the previous group. + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { + pOperator->status = OP_EXEC_DONE; + blockDataCleanup(pBlock); + + return PROJECT_RETRIEVE_DONE; + } + + // reset the value for a new group data + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; + + // existing rows that belongs to previous group. + if (pBlock->info.rows > 0) { + return PROJECT_RETRIEVE_DONE; + } + } + + // here we reach the start position, according to the limit/offset requirements. + + // set current group id + pLimitInfo->currentGroupId = pBlock->info.groupId; + + if (pLimitInfo->remainOffset >= pBlock->info.rows) { + pLimitInfo->remainOffset -= pBlock->info.rows; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; + } + + // check for the limitation in each group + if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pBlock, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { + pOperator->status = OP_EXEC_DONE; + } + + return PROJECT_RETRIEVE_DONE; + } + + // todo optimize performance + // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the + // they may not belong to the same group the limit/offset value is not valid in this case. + if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || + pLimitInfo->slimit.limit != -1) { + return PROJECT_RETRIEVE_DONE; + } else { // not full enough, continue to accumulate the output data in the buffer. + return PROJECT_RETRIEVE_CONTINUE; + } +} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7e955100c..9b1bc81dbe 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -24,7 +24,6 @@ #include "tdatablock.h" #include "tglobal.h" #include "tmsg.h" -#include "tsort.h" #include "ttime.h" #include "executorimpl.h" @@ -297,8 +296,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { colDataDestroy(pColData); } - typedef struct { bool hasAgg; int32_t numOfRows; @@ -1347,42 +1344,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { } } -// static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed -// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : -// pBlockInfo->window.skey + step; pQueryAttr->limit.offset = 0; return; -// } -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; -// } else { -// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; -// } -// -// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); -// -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value -// TSKEY *keys = (TSKEY *) pColInfoData->pData; -// -// // update the offset value -// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; -// pQueryAttr->limit.offset = 0; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, -// lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -// } - // void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // @@ -1723,159 +1684,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pInfo->pRes; } -int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length) { - if (result == NULL || length == NULL) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info); - SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo)); - int32_t size = tSimpleHashGetSize(pSup->pResultRowHashTable); - size_t keyLen = sizeof(uint64_t) * 2; // estimate the key length - int32_t totalSize = - sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); - - // no result - if (getTotalBufSize(pSup->pResultBuf) == 0) { - *result = NULL; - *length = 0; - return TSDB_CODE_SUCCESS; - } - - *result = (char*)taosMemoryCalloc(1, totalSize); - if (*result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t offset = sizeof(int32_t); - *(int32_t*)(*result + offset) = size; - offset += sizeof(int32_t); - - // prepare memory - SResultRowPosition* pos = &pInfo->resultRowInfo.cur; - void* pPage = getBufPage(pSup->pResultBuf, pos->pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); - setBufPageDirty(pPage, true); - releaseBufPage(pSup->pResultBuf, pPage); - - int32_t iter = 0; - void* pIter = NULL; - while ((pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter))) { - void* key = tSimpleHashGetKey(pIter, &keyLen); - SResultRowPosition* p1 = (SResultRowPosition*)pIter; - - pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId); - pRow = (SResultRow*)((char*)pPage + p1->offset); - setBufPageDirty(pPage, true); - releaseBufPage(pSup->pResultBuf, pPage); - - // recalculate the result size - int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize; - if (realTotalSize > totalSize) { - char* tmp = (char*)taosMemoryRealloc(*result, realTotalSize); - if (tmp == NULL) { - taosMemoryFree(*result); - *result = NULL; - return TSDB_CODE_OUT_OF_MEMORY; - } else { - *result = tmp; - } - } - // save key - *(int32_t*)(*result + offset) = keyLen; - offset += sizeof(int32_t); - memcpy(*result + offset, key, keyLen); - offset += keyLen; - - // save value - *(int32_t*)(*result + offset) = pSup->resultRowSize; - offset += sizeof(int32_t); - memcpy(*result + offset, pRow, pSup->resultRowSize); - offset += pSup->resultRowSize; - } - - *(int32_t*)(*result) = offset; - *length = offset; - - return TDB_CODE_SUCCESS; -} - -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { - if (pLimitInfo->remainGroupOffset > 0) { - if (pLimitInfo->currentGroupId == 0) { // it is the first group - pLimitInfo->currentGroupId = pBlock->info.groupId; - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { - // now it is the data from a new group - pLimitInfo->remainGroupOffset -= 1; - - // ignore data block in current group - if (pLimitInfo->remainGroupOffset > 0) { - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } - } - - // set current group id of the project operator - pLimitInfo->currentGroupId = pBlock->info.groupId; - } - - // here check for a new group data, we need to handle the data of the previous group. - if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { - pLimitInfo->numOfOutputGroups += 1; - if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - pOperator->status = OP_EXEC_DONE; - blockDataCleanup(pBlock); - - return PROJECT_RETRIEVE_DONE; - } - - // reset the value for a new group data - pLimitInfo->numOfOutputRows = 0; - pLimitInfo->remainOffset = pLimitInfo->limit.offset; - - // existing rows that belongs to previous group. - if (pBlock->info.rows > 0) { - return PROJECT_RETRIEVE_DONE; - } - } - - // here we reach the start position, according to the limit/offset requirements. - - // set current group id - pLimitInfo->currentGroupId = pBlock->info.groupId; - - if (pLimitInfo->remainOffset >= pBlock->info.rows) { - pLimitInfo->remainOffset -= pBlock->info.rows; - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { - blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); - pLimitInfo->remainOffset = 0; - } - - // check for the limitation in each group - if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { - int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); - blockDataKeepFirstNRows(pBlock, keepRows); - if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { - pOperator->status = OP_EXEC_DONE; - } - - return PROJECT_RETRIEVE_DONE; - } - - // todo optimize performance - // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the - // they may not belong to the same group the limit/offset value is not valid in this case. - if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || - pLimitInfo->slimit.limit != -1) { - return PROJECT_RETRIEVE_DONE; - } else { // not full enough, continue to accumulate the output data in the buffer. - return PROJECT_RETRIEVE_CONTINUE; - } -} - static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 26a5f6838d..fcfb79eb4b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -27,6 +27,36 @@ #include "thash.h" #include "ttypes.h" +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SExprSupp scalarSup; +} SGroupbyOperatorInfo; + +// The sort in partition may be needed later. +typedef struct SPartitionOperatorInfo { + SOptrBasicInfo binfo; + SArray* pGroupCols; + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SHashObj* pGroupSet; // quick locate the window object for each result + + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + int32_t groupIndex; // group index + int32_t pageIndex; // page index of current group + SExprSupp scalarSup; +} SPartitionOperatorInfo; + static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 4e1daac643..61dc7a1b76 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -24,6 +24,21 @@ #include "tmsg.h" #include "ttypes.h" +typedef struct SJoinOperatorInfo { + SSDataBlock* pRes; + int32_t joinType; + int32_t inputOrder; + + SSDataBlock* pLeft; + int32_t leftPos; + SColumnInfo leftCol; + + SSDataBlock* pRight; + int32_t rightPos; + SColumnInfo rightCol; + SNode* pCondAfterMerge; +} SJoinOperatorInfo; + static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index ce1d13775c..ada7964c67 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -17,6 +17,24 @@ #include "executorimpl.h" #include "functionMgt.h" +typedef struct SProjectOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SLimitInfo limitInfo; + bool mergeDataBlocks; + SSDataBlock* pFinalRes; +} SProjectOperatorInfo; + +typedef struct SIndefOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SExprSupp scalarSup; + uint64_t groupId; + SSDataBlock* pNextGroupRes; +} SIndefOperatorInfo; + static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 916b6df969..24ca3e2eed 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "executorimpl.h" #include "filter.h" #include "function.h" @@ -33,31 +32,17 @@ #include "ttypes.h" #include "vnode.h" +typedef struct SBlockDistInfo { + SSDataBlock* pResBlock; + STsdbReader* pHandle; + SReadHandle readHandle; + uint64_t uid; // table uid +} SBlockDistInfo; + #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); -static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, - size_t size, const char* dbName); -static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", - "ttl", "stable_name", "vgroup_id', 'uid", "type"}; - -static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; - -typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); -typedef int32_t (*__sys_check)(SNode* cond); - -typedef struct { - const char* name; - __sys_check chkFunc; - __sys_filte fltFunc; -} SSTabFltFuncDef; - -typedef struct { - void* pMeta; - void* pVnode; -} SSTabFltArg; typedef struct STableMergeScanExecInfo { SFileBlockLoadRecorder blockRecorder; @@ -71,54 +56,8 @@ typedef struct STableMergeScanSortSourceParam { SSDataBlock* inputBlock; } STableMergeScanSortSourceParam; -static int32_t sysChkFilter__Comm(SNode* pNode); -static int32_t sysChkFilter__DBName(SNode* pNode); -static int32_t sysChkFilter__VgroupId(SNode* pNode); -static int32_t sysChkFilter__TableName(SNode* pNode); -static int32_t sysChkFilter__CreateTime(SNode* pNode); -static int32_t sysChkFilter__Ncolumn(SNode* pNode); -static int32_t sysChkFilter__Ttl(SNode* pNode); -static int32_t sysChkFilter__STableName(SNode* pNode); -static int32_t sysChkFilter__Uid(SNode* pNode); -static int32_t sysChkFilter__Type(SNode* pNode); - -static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result); - -const SSTabFltFuncDef filterDict[] = { - {.name = "table_name", .chkFunc = sysChkFilter__TableName, .fltFunc = sysFilte__TableName}, - {.name = "db_name", .chkFunc = sysChkFilter__DBName, .fltFunc = sysFilte__DbName}, - {.name = "create_time", .chkFunc = sysChkFilter__CreateTime, .fltFunc = sysFilte__CreateTime}, - {.name = "columns", .chkFunc = sysChkFilter__Ncolumn, .fltFunc = sysFilte__Ncolumn}, - {.name = "ttl", .chkFunc = sysChkFilter__Ttl, .fltFunc = sysFilte__Ttl}, - {.name = "stable_name", .chkFunc = sysChkFilter__STableName, .fltFunc = sysFilte__STableName}, - {.name = "vgroup_id", .chkFunc = sysChkFilter__VgroupId, .fltFunc = sysFilte__VgroupId}, - {.name = "uid", .chkFunc = sysChkFilter__Uid, .fltFunc = sysFilte__Uid}, - {.name = "type", .chkFunc = sysChkFilter__Type, .fltFunc = sysFilte__Type}}; - -#define SYSTAB_FILTER_DICT_SIZE (sizeof(filterDict) / sizeof(filterDict[0])) - -static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result); -static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); -static int32_t optSysCheckOper(SNode* pOpear); -static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); - static bool processBlockWithProbability(const SSampleExecInfo* pInfo); -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, - SMetaReader* smrChildTable, const char* dbname, const char* tableName, - int32_t* pNumOfRows, const SSDataBlock* dataBlock); - -static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, - SFilterInfo* pFilterInfo); - bool processBlockWithProbability(const SSampleExecInfo* pInfo) { #if 0 if (pInfo->sampleRatio == 1) { @@ -2615,1638 +2554,6 @@ _error: return NULL; } -static void destroySysScanOperator(void* param) { - SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; - tsem_destroy(&pInfo->ready); - blockDataDestroy(pInfo->pRes); - - const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - } - if (pInfo->pIdx) { - taosArrayDestroy(pInfo->pIdx->uids); - taosMemoryFree(pInfo->pIdx); - pInfo->pIdx = NULL; - } - - taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFreeClear(pInfo->pUser); - - taosMemoryFreeClear(param); -} - -static int32_t getSysTableDbNameColId(const char* pTable) { - // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { - // return 1; - // } - return TSDB_INS_USER_STABLES_DBNAME_COLID; -} - -EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { - int32_t code = TSDB_CODE_SUCCESS; - ENodeType nType = nodeType(pNode); - - switch (nType) { - case QUERY_NODE_OPERATOR: { - SOperatorNode* node = (SOperatorNode*)pNode; - if (OP_TYPE_EQUAL == node->opType) { - *(int32_t*)pContext = 1; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_IGNORE_CHILD; - } - case QUERY_NODE_COLUMN: { - if (1 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SColumnNode* node = (SColumnNode*)pNode; - if (getSysTableDbNameColId(node->tableName) == node->colId) { - *(int32_t*)pContext = 2; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_CONTINUE; - } - case QUERY_NODE_VALUE: { - if (2 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SValueNode* node = (SValueNode*)pNode; - char* dbName = nodesGetValueFromNode(node); - strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); - *((char*)pContext + varDataLen(dbName)) = 0; - return DEAL_RES_END; // stop walk - } - default: - break; - } - return DEAL_RES_CONTINUE; -} - -static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { - if (NULL == pCondition) { - return; - } - nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName); -} - -static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { - SOperatorInfo* operator=(SOperatorInfo*) param; - SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; - if (TSDB_CODE_SUCCESS == code) { - pScanResInfo->pRsp = pMsg->pData; - - SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); - } else { - operator->pTaskInfo->code = code; - } - - tsem_post(&pScanResInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo) { - if (pFilterInfo == NULL) { - return pDataBlock->info.rows == 0 ? NULL : pDataBlock; - } - - doFilter(pDataBlock, pFilterInfo, NULL); - return pDataBlock->info.rows == 0 ? NULL : pDataBlock; -} - -static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { - size_t size = 0; - const SSysTableMeta* pMeta = NULL; - getInfosDbMeta(&pMeta, &size); - - int32_t index = 0; - for (int32_t i = 0; i < size; ++i) { - if (strcmp(pMeta[i].name, tableName) == 0) { - index = i; - break; - } - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = - createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); - blockDataAppendColInfo(pBlock, &colInfoData); - } - - return pBlock; -} - -int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { - int32_t n = 0; - - switch (type) { - case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); - break; - - case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); - break; - - case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t*)buf); - break; - - case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t*)buf); - break; - - case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t*)buf); - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t*)buf); - break; - - case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf)); - break; - - case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf)); - break; - - case TSDB_DATA_TYPE_BINARY: - if (bufSize < 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - memcpy(str, buf, bufSize); - n = bufSize; - break; - case TSDB_DATA_TYPE_NCHAR: - if (bufSize < 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); - if (length <= 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - n = length; - break; - case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%u", *(uint8_t*)buf); - break; - - case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%u", *(uint16_t*)buf); - break; - - case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t*)buf); - break; - - case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); - break; - - default: - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (len) *len = n; - - return TSDB_CODE_SUCCESS; -} - -static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { - SOperatorNode* node = (SOperatorNode*)pCond; - if (node->opType == OP_TYPE_EQUAL) { - if (nodeType(node->pLeft) == QUERY_NODE_COLUMN && - strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && - nodeType(node->pRight) == QUERY_NODE_VALUE) { - SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { - char* value = nodesGetValueFromNode(pValue); - strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); - return true; - } - } - } - return false; -} - -static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { - if (pCond == NULL) { - return false; - } - if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION) { - SLogicConditionNode* node = (SLogicConditionNode*)pCond; - if (LOGIC_COND_TYPE_AND == node->condType) { - SNode* pChild = NULL; - FOREACH(pChild, node->pParameterList) { - if (QUERY_NODE_OPERATOR == nodeType(pChild) && sysTableIsOperatorCondOnOneTable(pChild, condTable)) { - return true; - } - } - } - } - - if (QUERY_NODE_OPERATOR == nodeType(pCond)) { - return sysTableIsOperatorCondOnOneTable(pCond, condTable); - } - - return false; -} - -static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - char condTableName[TSDB_TABLE_NAME_LEN] = {0}; - // optimize when sql like where table_name='tablename' and xxx. - if (pInfo->pCondition && sysTableIsCondOnOneTable(pInfo->pCondition, condTableName)) { - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, condTableName); - - SMetaReader smrChildTable = {0}; - metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - return NULL; - } - - if (smrChildTable.me.type != TSDB_CHILD_TABLE) { - metaReaderClear(&smrChildTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - - SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByUid - return NULL; - } - - sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } - - int32_t ret = 0; - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { - continue; - } - - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - - SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&smrSuperTable); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - - metaReaderClear(&smrSuperTable); - - if (numOfRows >= pOperator->resultInfo.capacity) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - - blockDataDestroy(dataBlock); - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - setOperatorCompleted(pOperator); - } - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, - SFilterInfo* pFilterInfo) { - dataBlock->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false); - doFilterResult(pInfo->pRes, pFilterInfo); - blockDataCleanup(dataBlock); -} - -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, - SMetaReader* smrChildTable, const char* dbname, const char* tableName, - int32_t* pNumOfRows, const SSDataBlock* dataBlock) { - char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); - - int32_t numOfRows = *pNumOfRows; - - int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; - for (int32_t i = 0; i < numOfTags; ++i) { - SColumnInfoData* pColInfoData = NULL; - - // table name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tableName, false); - - // database name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // super table name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, stableName, false); - - // tag name - char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, tagName, false); - - // tag type - int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); - char tagTypeStr[VARSTR_HEADER_SIZE + 32]; - int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); - if (tagType == TSDB_DATA_TYPE_VARCHAR) { - tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); - } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += sprintf( - varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); - } - varDataSetLen(tagTypeStr, tagTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); - - STagVal tagVal = {0}; - tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; - char* tagData = NULL; - uint32_t tagLen = 0; - - if (tagType == TSDB_DATA_TYPE_JSON) { - tagData = (char*)smrChildTable->me.ctbEntry.pTags; - } else { - bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); - if (exist) { - if (IS_VAR_DATA_TYPE(tagType)) { - tagData = (char*)tagVal.pData; - tagLen = tagVal.nData; - } else { - tagData = (char*)&tagVal.i64; - tagLen = tDataTypes[tagType].bytes; - } - } - } - - char* tagVarChar = NULL; - if (tagData != NULL) { - if (tagType == TSDB_DATA_TYPE_JSON) { - char* tagJson = parseTagDatatoJson(tagData); - tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); - memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); - varDataSetLen(tagVarChar, strlen(tagJson)); - taosMemoryFree(tagJson); - } else { - int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) - : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); - tagVarChar = taosMemoryMalloc(bufSize); - int32_t len = -1; - convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); - varDataSetLen(tagVarChar, len); - } - } - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, tagVarChar, - (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); - taosMemoryFree(tagVarChar); - ++numOfRows; - } - - *pNumOfRows = numOfRows; - - return TSDB_CODE_SUCCESS; -} - -typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); - -int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { - int32_t cmp = func(a, b); - switch (comparType) { - case OP_TYPE_LOWER_THAN: - if (cmp < 0) return 0; - break; - case OP_TYPE_LOWER_EQUAL: { - if (cmp <= 0) return 0; - break; - } - case OP_TYPE_GREATER_THAN: { - if (cmp > 0) return 0; - break; - } - case OP_TYPE_GREATER_EQUAL: { - if (cmp >= 0) return 0; - break; - } - case OP_TYPE_EQUAL: { - if (cmp == 0) return 0; - break; - } - default: - return -1; - } - return cmp; -} - -static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); -} -static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); -} -static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); -} -static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); -} -static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); -} - -static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); -} -static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { - if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { - *reverse = true; - } - if (ctype == OP_TYPE_LOWER_THAN) - return optSysFilterFuncImpl__LowerThan; - else if (ctype == OP_TYPE_LOWER_EQUAL) - return optSysFilterFuncImpl__LowerEqual; - else if (ctype == OP_TYPE_GREATER_THAN) - return optSysFilterFuncImpl__GreaterThan; - else if (ctype == OP_TYPE_GREATER_EQUAL) - return optSysFilterFuncImpl__GreaterEqual; - else if (ctype == OP_TYPE_EQUAL) - return optSysFilterFuncImpl__Equal; - else if (ctype == OP_TYPE_NOT_EQUAL) - return optSysFilterFuncImpl__NoEqual; - return NULL; -} -static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; - - const char* db = NULL; - vnodeGetInfo(pVnode, &db, NULL); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); - if (ret == 0) return 0; - - return -2; -} -static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; - - int64_t vgId = 0; - vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); - if (ret == 0) return 0; - - return -1; -} -static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - SMetaFltParam param = {.suid = 0, - .cid = 0, - .type = TSDB_DATA_TYPE_VARCHAR, - .val = pVal->datum.p, - .reverse = reverse, - .filterFunc = func}; - return -1; -} - -static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - SMetaFltParam param = {.suid = 0, - .cid = 0, - .type = TSDB_DATA_TYPE_BIGINT, - .val = &pVal->datum.i, - .reverse = reverse, - .filterFunc = func}; - - int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); - return ret; -} -static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} - -static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysChkFilter__Comm(SNode* pNode) { - // impl - SOperatorNode* pOper = (SOperatorNode*)pNode; - EOperatorType opType = pOper->opType; - if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && - opType != OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { - return -1; - } - return 0; -} - -static int32_t sysChkFilter__DBName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - - if (pOper->opType != OP_TYPE_EQUAL && pOper->opType != OP_TYPE_NOT_EQUAL) { - return -1; - } - - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - - return 0; -} -static int32_t sysChkFilter__VgroupId(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__TableName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__CreateTime(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} - -static int32_t sysChkFilter__Ncolumn(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Ttl(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__STableName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Uid(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Type(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { - if (optSysCheckOper(cond) != 0) return -1; - - SOperatorNode* pNode = (SOperatorNode*)cond; - - int8_t i = 0; - for (; i < SYSTAB_FILTER_DICT_SIZE; i++) { - if (strcmp(filterDict[i].name, ((SColumnNode*)(pNode->pLeft))->colName) == 0) { - break; - } - } - if (i >= SYSTAB_FILTER_DICT_SIZE) return -1; - - if (filterDict[i].chkFunc(cond) != 0) return -1; - - return filterDict[i].fltFunc(arg, cond, result); -} - -static int32_t optSysCheckOper(SNode* pOpear) { - if (nodeType(pOpear) != QUERY_NODE_OPERATOR) return -1; - - SOperatorNode* pOper = (SOperatorNode*)pOpear; - if (pOper->opType < OP_TYPE_GREATER_THAN || pOper->opType > OP_TYPE_NOT_EQUAL) { - return -1; - } - - if (nodeType(pOper->pLeft) != QUERY_NODE_COLUMN || nodeType(pOper->pRight) != QUERY_NODE_VALUE) { - return -1; - } - return 0; -} - -static int tableUidCompare(const void* a, const void* b) { - int64_t u1 = *(int64_t*)a; - int64_t u2 = *(int64_t*)b; - if (u1 == u2) { - return 0; - } - return u1 < u2 ? -1 : 1; -} - -typedef struct MergeIndex { - int idx; - int len; -} MergeIndex; - -static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) { - uint64_t v; - int32_t m; - while (s <= e) { - m = s + (e - s) / 2; - v = *(uint64_t*)taosArrayGet(arr, m); - if (v >= k) { - e = m - 1; - } else { - s = m + 1; - } - } - return s; -} - -void optSysIntersection(SArray* in, SArray* out) { - int32_t sz = (int32_t)taosArrayGetSize(in); - if (sz <= 0) { - return; - } - MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); - for (int i = 0; i < sz; i++) { - SArray* t = taosArrayGetP(in, i); - mi[i].len = (int32_t)taosArrayGetSize(t); - mi[i].idx = 0; - } - - SArray* base = taosArrayGetP(in, 0); - for (int i = 0; i < taosArrayGetSize(base); i++) { - uint64_t tgt = *(uint64_t*)taosArrayGet(base, i); - bool has = true; - for (int j = 1; j < taosArrayGetSize(in); j++) { - SArray* oth = taosArrayGetP(in, j); - int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); - if (mid >= 0 && mid < mi[j].len) { - uint64_t val = *(uint64_t*)taosArrayGet(oth, mid); - has = (val == tgt ? true : false); - mi[j].idx = mid; - } else { - has = false; - } - } - if (has == true) { - taosArrayPush(out, &tgt); - } - } - taosMemoryFreeClear(mi); -} - -static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { - // TODO, find comm mem from mRslt - for (int i = 0; i < taosArrayGetSize(mRslt); i++) { - SArray* arslt = taosArrayGetP(mRslt, i); - taosArraySort(arslt, tableUidCompare); - } - optSysIntersection(mRslt, rslt); - return 0; -} - -static int32_t optSysSpecialColumn(SNode* cond) { - SOperatorNode* pOper = (SOperatorNode*)cond; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) { - if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) { - return 1; - } - } - return 0; -} - -static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { - int ret = -1; - if (nodeType(cond) == QUERY_NODE_OPERATOR) { - ret = optSysTabFilteImpl(arg, cond, result); - if (ret == 0) { - SOperatorNode* pOper = (SOperatorNode*)cond; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - if (0 == strcmp(pCol->colName, "create_time")) { - return 0; - } - return -1; - } - return ret; - } - - if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { - return ret; - } - - SLogicConditionNode* pNode = (SLogicConditionNode*)cond; - SNodeList* pList = (SNodeList*)pNode->pParameterList; - - int32_t len = LIST_LENGTH(pList); - - bool hasIdx = false; - bool hasRslt = true; - SArray* mRslt = taosArrayInit(len, POINTER_BYTES); - - SListCell* cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - - SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); - - ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); - if (ret == 0) { - // has index - hasIdx = true; - if (optSysSpecialColumn(cell->pNode) == 0) { - taosArrayPush(mRslt, &aRslt); - } else { - // db_name/vgroup not result - taosArrayDestroy(aRslt); - } - } else if (ret == -2) { - // current vg - hasIdx = true; - hasRslt = false; - taosArrayDestroy(aRslt); - break; - } else { - taosArrayDestroy(aRslt); - } - cell = cell->pNext; - } - if (hasRslt && hasIdx) { - optSysMergeRslt(mRslt, result); - } - - for (int i = 0; i < taosArrayGetSize(mRslt); i++) { - SArray* aRslt = taosArrayGetP(mRslt, i); - taosArrayDestroy(aRslt); - } - taosArrayDestroy(mRslt); - if (hasRslt == false) { - return -2; - } - if (hasRslt && hasIdx) { - cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - SOperatorNode* pOper = (SOperatorNode*)cell->pNode; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - if (0 == strcmp(pCol->colName, "create_time")) { - return 0; - } - cell = cell->pNext; - } - return -1; - } - return -1; -} - -static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - SSysTableIndex* pIdx = pInfo->pIdx; - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - int ret = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t i = pIdx->lastIdx; - for (; i < taosArrayGetSize(pIdx->uids); i++) { - tb_uid_t* uid = taosArrayGet(pIdx->uids, i); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - ret = metaGetTableEntryByUid(&mr, *uid); - if (ret < 0) { - metaReaderClear(&mr); - continue; - } - STR_TO_VARSTR(n, mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr1 = {0}; - metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); - - int64_t suid = mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr1, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr1); - metaReaderClear(&mr); - T_LONG_JMP(pTaskInfo->env, terrno); - } - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr1.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr1); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - // impl later - } - - metaReaderClear(&mr); - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - } - - if (i >= taosArrayGetSize(pIdx->uids)) { - setOperatorCompleted(pOperator); - } else { - pIdx->lastIdx = i + 1; - } - - blockDataDestroy(p); - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SSysTableScanInfo* pInfo = pOperator->info; - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = pInfo->pCur->mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); - - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - } - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - } - - blockDataDestroy(p); - - // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - setOperatorCompleted(pOperator); - } - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - SNode* pCondition = pInfo->pCondition; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. - if (pInfo->readHandle.mnd != NULL) { - buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { - if (pInfo->showRewrite == false) { - if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; - - SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); - idx->init = 0; - idx->uids = taosArrayInit(128, sizeof(int64_t)); - idx->lastIdx = 0; - - pInfo->pIdx = idx; // set idx arg - - int flt = optSysTabFilte(&arg, pCondition, idx->uids); - if (flt == 0) { - pInfo->pIdx->init = 1; - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); - return blk; - } else if (flt == -2) { - qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); - return NULL; - } else if (flt == -1) { - // not idx - qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); - } - } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); - return blk; - } - } - - return sysTableBuildUserTables(pOperator); - } - return NULL; -} - -static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - pInfo->pRes->info.rows = 0; - pOperator->status = OP_EXEC_DONE; - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { - // build message and send to mnode to fetch the content of system tables. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - char dbName[TSDB_DB_NAME_LEN] = {0}; - - const char* name = tNameGetTableName(&pInfo->name); - if (pInfo->showRewrite) { - getDBNameFromCondition(pInfo->pCondition, dbName); - sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); - } - - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTables(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTags(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && - IS_SYS_DBNAME(dbName)) { - return sysTableScanUserSTables(pOperator); - } else { // load the meta from mnode of the given epset - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - int64_t startTs = taosGetTimestampUs(); - tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); - - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = taosMemoryCalloc(1, contLen); - tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE - : TDMT_MND_SYSTABLE_RETRIEVE; - - pMsgSendInfo->param = pOperator; - pMsgSendInfo->msgInfo.pData = buf1; - pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = loadSysTableCallback; - pMsgSendInfo->requestId = pTaskInfo->id.queryId; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); - - if (pTaskInfo->code) { - qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), - pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); - return NULL; - } - - SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; - pInfo->req.showId = pRsp->handle; - - if (pRsp->numOfRows == 0 || pRsp->completed) { - pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), - pRsp->numOfRows, pInfo->loadInfo.totalRows); - - if (pRsp->numOfRows == 0) { - taosMemoryFree(pRsp); - return NULL; - } - } - - char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); - updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - - // todo log the filter info - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - taosMemoryFree(pRsp); - if (pInfo->pRes->info.rows > 0) { - return pInfo->pRes; - } else if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - } - } -} - -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, capacity); - - size_t size = 0; - const SSysTableMeta* pSysDbTableMeta = NULL; - - getInfosDbMeta(&pSysDbTableMeta, &size); - p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); - - getPerfDbMeta(&pSysDbTableMeta, &size); - p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); - - pInfo->pRes->info.rows = p->info.rows; - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - blockDataDestroy(p); - - return pInfo->pRes->info.rows; -} - -int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, - const char* dbName) { - char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t numOfRows = p->info.rows; - - for (int32_t i = 0; i < size; ++i) { - const SSysTableMeta* pm = &pSysDbTableMeta[i]; - if (!sysInfo && pm->sysInfo) { - continue; - } - - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - - STR_TO_VARSTR(n, pm->name); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - STR_TO_VARSTR(n, dbName); - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, n, false); - - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppendNULL(pColInfoData, numOfRows); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); - - for (int32_t j = 4; j <= 8; ++j) { - pColInfoData = taosArrayGet(p->pDataBlock, j); - colDataAppendNULL(pColInfoData, numOfRows); - } - - STR_TO_VARSTR(n, "SYSTEM_TABLE"); - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - numOfRows += 1; - } - - return numOfRows; -} - -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, - const char* pUser, SExecTaskInfo* pTaskInfo) { - SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SScanPhysiNode* pScanNode = &pScanPhyNode->scan; - SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; - - int32_t num = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->accountId = pScanPhyNode->accountId; - pInfo->pUser = taosMemoryStrDup((void*)pUser); - pInfo->sysInfo = pScanPhyNode->sysInfo; - pInfo->showRewrite = pScanPhyNode->showRewrite; - pInfo->pRes = createResDataBlock(pDescNode); - - pInfo->pCondition = pScanNode->node.pConditions; - code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - initResultSizeInfo(&pOperator->resultInfo, 4096); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - - tNameAssign(&pInfo->name, &pScanNode->tableName); - const char* name = tNameGetTableName(&pInfo->name); - - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - pInfo->readHandle = *(SReadHandle*)readHandle; - } else { - tsem_init(&pInfo->ready, 0, 0); - pInfo->epSet = pScanPhyNode->mgmtEpSet; - pInfo->readHandle = *(SReadHandle*)readHandle; - } - - setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, - pInfo, pTaskInfo); - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - destroySysScanOperator(pInfo); - } - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -4271,7 +2578,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pInfo->pTableList, pInfo->curPos); + STableKeyInfo* item = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { @@ -4358,7 +2665,6 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; pInfo->pRes = createResDataBlock(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 14e3163455..0efa27dd46 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -17,6 +17,18 @@ #include "executorimpl.h" #include "tdatablock.h" +typedef struct SSortOperatorInfo { + SOptrBasicInfo binfo; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SColMatchInfo matchInfo; + int32_t bufPageSize; + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + SLimitInfo limitInfo; +} SSortOperatorInfo; + static SSDataBlock* doSort(SOperatorInfo* pOperator); static int32_t doOpenSortOperator(SOperatorInfo* pOperator); static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c new file mode 100644 index 0000000000..f6a0c57b66 --- /dev/null +++ b/source/libs/executor/src/sysscanoperator.c @@ -0,0 +1,1774 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executorimpl.h" +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "systable.h" +#include "tname.h" +#include "ttime.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); +typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); +typedef int32_t (*__sys_check)(SNode* cond); + +typedef struct SSTabFltArg { + void* pMeta; + void* pVnode; +} SSTabFltArg; + +typedef struct SSysTableIndex { + int8_t init; + SArray* uids; + int32_t lastIdx; +} SSysTableIndex; + +typedef struct SSysTableScanInfo { + SRetrieveMetaTableRsp* pRsp; + SRetrieveTableReq req; + SEpSet epSet; + tsem_t ready; + SReadHandle readHandle; + int32_t accountId; + const char* pUser; + bool sysInfo; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + SMTbCursor* pCur; // cursor for iterate the local table meta store. + SSysTableIndex* pIdx; // idx for local table meta + SColMatchInfo matchInfo; + SName name; + SSDataBlock* pRes; + int64_t numOfBlocks; // extract basic running information. + SLoadRemoteDataInfo loadInfo; +} SSysTableScanInfo; + +typedef struct { + const char* name; + __sys_check chkFunc; + __sys_filte fltFunc; +} SSTabFltFuncDef; + +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int32_t sysChkFilter__Comm(SNode* pNode); +static int32_t sysChkFilter__DBName(SNode* pNode); +static int32_t sysChkFilter__VgroupId(SNode* pNode); +static int32_t sysChkFilter__TableName(SNode* pNode); +static int32_t sysChkFilter__CreateTime(SNode* pNode); +static int32_t sysChkFilter__Ncolumn(SNode* pNode); +static int32_t sysChkFilter__Ttl(SNode* pNode); +static int32_t sysChkFilter__STableName(SNode* pNode); +static int32_t sysChkFilter__Uid(SNode* pNode); +static int32_t sysChkFilter__Type(SNode* pNode); + +static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result); + +const SSTabFltFuncDef filterDict[] = { + {.name = "table_name", .chkFunc = sysChkFilter__TableName, .fltFunc = sysFilte__TableName}, + {.name = "db_name", .chkFunc = sysChkFilter__DBName, .fltFunc = sysFilte__DbName}, + {.name = "create_time", .chkFunc = sysChkFilter__CreateTime, .fltFunc = sysFilte__CreateTime}, + {.name = "columns", .chkFunc = sysChkFilter__Ncolumn, .fltFunc = sysFilte__Ncolumn}, + {.name = "ttl", .chkFunc = sysChkFilter__Ttl, .fltFunc = sysFilte__Ttl}, + {.name = "stable_name", .chkFunc = sysChkFilter__STableName, .fltFunc = sysFilte__STableName}, + {.name = "vgroup_id", .chkFunc = sysChkFilter__VgroupId, .fltFunc = sysFilte__VgroupId}, + {.name = "uid", .chkFunc = sysChkFilter__Uid, .fltFunc = sysFilte__Uid}, + {.name = "type", .chkFunc = sysChkFilter__Type, .fltFunc = sysFilte__Type}}; + +#define SYSTAB_FILTER_DICT_SIZE (sizeof(filterDict) / sizeof(filterDict[0])) + +static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, + size_t size, const char* dbName); + +static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", + "ttl", "stable_name", "vgroup_id', 'uid", "type"}; + +static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; + +static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName); +static void destroySysScanOperator(void* param); +static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code); +static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo); +static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse); + +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock); + +static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, + SFilterInfo* pFilterInfo); + +int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { + void* pVnode = ((SSTabFltArg*)arg)->pVnode; + + const char* db = NULL; + vnodeGetInfo(pVnode, &db, NULL); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + bool reverse = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); + if (ret == 0) return 0; + + return -2; +} + +int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { + void* pVnode = ((SSTabFltArg*)arg)->pVnode; + + int64_t vgId = 0; + vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); + if (ret == 0) return 0; + + return -1; +} + +int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + SMetaFltParam param = {.suid = 0, + .cid = 0, + .type = TSDB_DATA_TYPE_VARCHAR, + .val = pVal->datum.p, + .reverse = reverse, + .filterFunc = func}; + return -1; +} + +int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + SMetaFltParam param = {.suid = 0, + .cid = 0, + .type = TSDB_DATA_TYPE_BIGINT, + .val = &pVal->datum.i, + .reverse = reverse, + .filterFunc = func}; + + int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); + return ret; +} + +int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { + int32_t cmp = func(a, b); + switch (comparType) { + case OP_TYPE_LOWER_THAN: + if (cmp < 0) return 0; + break; + case OP_TYPE_LOWER_EQUAL: { + if (cmp <= 0) return 0; + break; + } + case OP_TYPE_GREATER_THAN: { + if (cmp > 0) return 0; + break; + } + case OP_TYPE_GREATER_EQUAL: { + if (cmp >= 0) return 0; + break; + } + case OP_TYPE_EQUAL: { + if (cmp == 0) return 0; + break; + } + default: + return -1; + } + return cmp; +} + +static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); +} +static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); +} +static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); +} +static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); +} +static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); +} + +static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); +} + +static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result); +static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); +static int32_t optSysCheckOper(SNode* pOpear); +static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); + +__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { + if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { + *reverse = true; + } + if (ctype == OP_TYPE_LOWER_THAN) + return optSysFilterFuncImpl__LowerThan; + else if (ctype == OP_TYPE_LOWER_EQUAL) + return optSysFilterFuncImpl__LowerEqual; + else if (ctype == OP_TYPE_GREATER_THAN) + return optSysFilterFuncImpl__GreaterThan; + else if (ctype == OP_TYPE_GREATER_EQUAL) + return optSysFilterFuncImpl__GreaterEqual; + else if (ctype == OP_TYPE_EQUAL) + return optSysFilterFuncImpl__Equal; + else if (ctype == OP_TYPE_NOT_EQUAL) + return optSysFilterFuncImpl__NoEqual; + return NULL; +} + +static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { + SOperatorNode* node = (SOperatorNode*)pCond; + if (node->opType == OP_TYPE_EQUAL) { + if (nodeType(node->pLeft) == QUERY_NODE_COLUMN && + strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && + nodeType(node->pRight) == QUERY_NODE_VALUE) { + SValueNode* pValue = (SValueNode*)node->pRight; + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || + pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + char* value = nodesGetValueFromNode(pValue); + strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); + return true; + } + } + } + return false; +} + +static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { + if (pCond == NULL) { + return false; + } + if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION) { + SLogicConditionNode* node = (SLogicConditionNode*)pCond; + if (LOGIC_COND_TYPE_AND == node->condType) { + SNode* pChild = NULL; + FOREACH(pChild, node->pParameterList) { + if (QUERY_NODE_OPERATOR == nodeType(pChild) && sysTableIsOperatorCondOnOneTable(pChild, condTable)) { + return true; + } + } + } + } + + if (QUERY_NODE_OPERATOR == nodeType(pCond)) { + return sysTableIsOperatorCondOnOneTable(pCond, condTable); + } + + return false; +} + +static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + char condTableName[TSDB_TABLE_NAME_LEN] = {0}; + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->pCondition && sysTableIsCondOnOneTable(pInfo->pCondition, condTableName)) { + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, condTableName); + + SMetaReader smrChildTable = {0}; + metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { + metaReaderClear(&smrChildTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); + metaReaderClear(&smrSuperTable); + metaReaderClear(&smrChildTable); + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } + + int32_t ret = 0; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { + continue; + } + + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&smrSuperTable); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } + + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); + + metaReaderClear(&smrSuperTable); + + if (numOfRows >= pOperator->resultInfo.capacity) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + blockDataDestroy(dataBlock); + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, + SFilterInfo* pFilterInfo) { + dataBlock->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false); + doFilterResult(pInfo->pRes, pFilterInfo); + blockDataCleanup(dataBlock); +} + +int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + if (bufSize < 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + memcpy(str, buf, bufSize); + n = bufSize; + break; + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); + if (length <= 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + n = length; + break; + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%u", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%u", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + + default: + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (len) *len = n; + + return TSDB_CODE_SUCCESS; +} + +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock) { + char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); + + int32_t numOfRows = *pNumOfRows; + + int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; + for (int32_t i = 0; i < numOfTags; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tableName, false); + + // database name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // super table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, stableName, false); + + // tag name + char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, tagName, false); + + // tag type + int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + char tagTypeStr[VARSTR_HEADER_SIZE + 32]; + int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); + if (tagType == TSDB_DATA_TYPE_VARCHAR) { + tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + } else if (tagType == TSDB_DATA_TYPE_NCHAR) { + tagTypeLen += sprintf( + varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(tagTypeStr, tagTypeLen); + colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); + + STagVal tagVal = {0}; + tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; + char* tagData = NULL; + uint32_t tagLen = 0; + + if (tagType == TSDB_DATA_TYPE_JSON) { + tagData = (char*)smrChildTable->me.ctbEntry.pTags; + } else { + bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); + if (exist) { + if (IS_VAR_DATA_TYPE(tagType)) { + tagData = (char*)tagVal.pData; + tagLen = tagVal.nData; + } else { + tagData = (char*)&tagVal.i64; + tagLen = tDataTypes[tagType].bytes; + } + } + } + + char* tagVarChar = NULL; + if (tagData != NULL) { + if (tagType == TSDB_DATA_TYPE_JSON) { + char* tagJson = parseTagDatatoJson(tagData); + tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); + memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); + varDataSetLen(tagVarChar, strlen(tagJson)); + taosMemoryFree(tagJson); + } else { + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) + : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); + tagVarChar = taosMemoryMalloc(bufSize); + int32_t len = -1; + convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); + varDataSetLen(tagVarChar, len); + } + } + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, tagVarChar, + (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); + taosMemoryFree(tagVarChar); + ++numOfRows; + } + + *pNumOfRows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { + size_t size = 0; + const SSysTableMeta* pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for (int32_t i = 0; i < size; ++i) { + if (strcmp(pMeta[i].name, tableName) == 0) { + index = i; + break; + } + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = + createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); + } + + return pBlock; +} + +int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, + const char* dbName) { + char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t numOfRows = p->info.rows; + + for (int32_t i = 0; i < size; ++i) { + const SSysTableMeta* pm = &pSysDbTableMeta[i]; + if (!sysInfo && pm->sysInfo) { + continue; + } + + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + + STR_TO_VARSTR(n, pm->name); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + STR_TO_VARSTR(n, dbName); + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, n, false); + + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppendNULL(pColInfoData, numOfRows); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); + + for (int32_t j = 4; j <= 8; ++j) { + pColInfoData = taosArrayGet(p->pDataBlock, j); + colDataAppendNULL(pColInfoData, numOfRows); + } + + STR_TO_VARSTR(n, "SYSTEM_TABLE"); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + numOfRows += 1; + } + + return numOfRows; +} + +int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, capacity); + + size_t size = 0; + const SSysTableMeta* pSysDbTableMeta = NULL; + + getInfosDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); + + getPerfDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); + + pInfo->pRes->info.rows = p->info.rows; + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + blockDataDestroy(p); + + return pInfo->pRes->info.rows; +} + +static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + SSysTableIndex* pIdx = pInfo->pIdx; + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + int ret = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t i = pIdx->lastIdx; + for (; i < taosArrayGetSize(pIdx->uids); i++) { + tb_uid_t* uid = taosArrayGet(pIdx->uids, i); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + ret = metaGetTableEntryByUid(&mr, *uid); + if (ret < 0) { + metaReaderClear(&mr); + continue; + } + STR_TO_VARSTR(n, mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr1 = {0}; + metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + + int64_t suid = mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr1, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr1); + metaReaderClear(&mr); + T_LONG_JMP(pTaskInfo->env, terrno); + } + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr1.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr1); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + // impl later + } + + metaReaderClear(&mr); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + if (i >= taosArrayGetSize(pIdx->uids)) { + setOperatorCompleted(pOperator); + } else { + pIdx->lastIdx = i + 1; + } + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SSysTableScanInfo* pInfo = pOperator->info; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + blockDataDestroy(p); + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + SNode* pCondition = pInfo->pCondition; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. + if (pInfo->readHandle.mnd != NULL) { + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + if (pInfo->showRewrite == false) { + if (pCondition != NULL && pInfo->pIdx == NULL) { + SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; + + SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); + idx->init = 0; + idx->uids = taosArrayInit(128, sizeof(int64_t)); + idx->lastIdx = 0; + + pInfo->pIdx = idx; // set idx arg + + int flt = optSysTabFilte(&arg, pCondition, idx->uids); + if (flt == 0) { + pInfo->pIdx->init = 1; + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } else if (flt == -2) { + qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); + return NULL; + } else if (flt == -1) { + // not idx + qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); + } + } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } + } + + return sysTableBuildUserTables(pOperator); + } + return NULL; +} + +static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + pInfo->pRes->info.rows = 0; + pOperator->status = OP_EXEC_DONE; + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static int32_t getSysTableDbNameColId(const char* pTable) { + // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { + // return 1; + // } + return TSDB_INS_USER_STABLES_DBNAME_COLID; +} + +static EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + ENodeType nType = nodeType(pNode); + + switch (nType) { + case QUERY_NODE_OPERATOR: { + SOperatorNode* node = (SOperatorNode*)pNode; + if (OP_TYPE_EQUAL == node->opType) { + *(int32_t*)pContext = 1; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_COLUMN: { + if (1 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SColumnNode* node = (SColumnNode*)pNode; + if (getSysTableDbNameColId(node->tableName) == node->colId) { + *(int32_t*)pContext = 2; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_CONTINUE; + } + case QUERY_NODE_VALUE: { + if (2 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SValueNode* node = (SValueNode*)pNode; + char* dbName = nodesGetValueFromNode(node); + strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); + *((char*)pContext + varDataLen(dbName)) = 0; + return DEAL_RES_END; // stop walk + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { + if (NULL == pCondition) { + return; + } + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName); +} + +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { + // build message and send to mnode to fetch the content of system tables. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + char dbName[TSDB_DB_NAME_LEN] = {0}; + + const char* name = tNameGetTableName(&pInfo->name); + if (pInfo->showRewrite) { + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } + + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTables(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTags(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && + IS_SYS_DBNAME(dbName)) { + return sysTableScanUserSTables(pOperator); + } else { // load the meta from mnode of the given epset + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + int64_t startTs = taosGetTimestampUs(); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); + + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; + + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = loadSysTableCallback; + pMsgSendInfo->requestId = pTaskInfo->id.queryId; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pInfo->ready); + + if (pTaskInfo->code) { + qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), + pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); + return NULL; + } + + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), + pRsp->numOfRows, pInfo->loadInfo.totalRows); + + if (pRsp->numOfRows == 0) { + taosMemoryFree(pRsp); + return NULL; + } + } + + char* pStart = pRsp->data; + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); + + // todo log the filter info + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + taosMemoryFree(pRsp); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } else if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + } + } +} + +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, + const char* pUser, SExecTaskInfo* pTaskInfo) { + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SScanPhysiNode* pScanNode = &pScanPhyNode->scan; + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + + int32_t num = 0; + int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->accountId = pScanPhyNode->accountId; + pInfo->pUser = taosMemoryStrDup((void*)pUser); + pInfo->sysInfo = pScanPhyNode->sysInfo; + pInfo->showRewrite = pScanPhyNode->showRewrite; + pInfo->pRes = createResDataBlock(pDescNode); + + pInfo->pCondition = pScanNode->node.pConditions; + code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + tNameAssign(&pInfo->name, &pScanNode->tableName); + const char* name = tNameGetTableName(&pInfo->name); + + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + pInfo->readHandle = *(SReadHandle*)readHandle; + } else { + tsem_init(&pInfo->ready, 0, 0); + pInfo->epSet = pScanPhyNode->mgmtEpSet; + pInfo->readHandle = *(SReadHandle*)readHandle; + } + + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + destroySysScanOperator(pInfo); + } + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroySysScanOperator(void* param) { + SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; + tsem_destroy(&pInfo->ready); + blockDataDestroy(pInfo->pRes); + + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + } + if (pInfo->pIdx) { + taosArrayDestroy(pInfo->pIdx->uids); + taosMemoryFree(pInfo->pIdx); + pInfo->pIdx = NULL; + } + + taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFreeClear(pInfo->pUser); + + taosMemoryFreeClear(param); +} + +int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { + SOperatorInfo* operator=(SOperatorInfo*) param; + SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; + if (TSDB_CODE_SUCCESS == code) { + pScanResInfo->pRsp = pMsg->pData; + + SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); + } else { + operator->pTaskInfo->code = code; + } + + tsem_post(&pScanResInfo->ready); + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo) { + if (pFilterInfo == NULL) { + return pDataBlock->info.rows == 0 ? NULL : pDataBlock; + } + + doFilter(pDataBlock, pFilterInfo, NULL); + return pDataBlock->info.rows == 0 ? NULL : pDataBlock; +} + +static int32_t sysChkFilter__Comm(SNode* pNode) { + // impl + SOperatorNode* pOper = (SOperatorNode*)pNode; + EOperatorType opType = pOper->opType; + if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && + opType != OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { + return -1; + } + return 0; +} + +static int32_t sysChkFilter__DBName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + + if (pOper->opType != OP_TYPE_EQUAL && pOper->opType != OP_TYPE_NOT_EQUAL) { + return -1; + } + + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + + return 0; +} +static int32_t sysChkFilter__VgroupId(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__TableName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__CreateTime(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} + +static int32_t sysChkFilter__Ncolumn(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Ttl(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__STableName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Uid(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Type(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { + if (optSysCheckOper(cond) != 0) return -1; + + SOperatorNode* pNode = (SOperatorNode*)cond; + + int8_t i = 0; + for (; i < SYSTAB_FILTER_DICT_SIZE; i++) { + if (strcmp(filterDict[i].name, ((SColumnNode*)(pNode->pLeft))->colName) == 0) { + break; + } + } + if (i >= SYSTAB_FILTER_DICT_SIZE) return -1; + + if (filterDict[i].chkFunc(cond) != 0) return -1; + + return filterDict[i].fltFunc(arg, cond, result); +} + +static int32_t optSysCheckOper(SNode* pOpear) { + if (nodeType(pOpear) != QUERY_NODE_OPERATOR) return -1; + + SOperatorNode* pOper = (SOperatorNode*)pOpear; + if (pOper->opType < OP_TYPE_GREATER_THAN || pOper->opType > OP_TYPE_NOT_EQUAL) { + return -1; + } + + if (nodeType(pOper->pLeft) != QUERY_NODE_COLUMN || nodeType(pOper->pRight) != QUERY_NODE_VALUE) { + return -1; + } + return 0; +} + +static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t*)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void optSysIntersection(SArray* in, SArray* out) { + int32_t sz = (int32_t)taosArrayGetSize(in); + if (sz <= 0) { + return; + } + MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray* t = taosArrayGetP(in, i); + mi[i].len = (int32_t)taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray* base = taosArrayGetP(in, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t*)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(in); j++) { + SArray* oth = taosArrayGetP(in, j); + int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t*)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(out, &tgt); + } + } + taosMemoryFreeClear(mi); +} + +static int tableUidCompare(const void* a, const void* b) { + int64_t u1 = *(int64_t*)a; + int64_t u2 = *(int64_t*)b; + if (u1 == u2) { + return 0; + } + return u1 < u2 ? -1 : 1; +} + +static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { + // TODO, find comm mem from mRslt + for (int i = 0; i < taosArrayGetSize(mRslt); i++) { + SArray* arslt = taosArrayGetP(mRslt, i); + taosArraySort(arslt, tableUidCompare); + } + optSysIntersection(mRslt, rslt); + return 0; +} + +static int32_t optSysSpecialColumn(SNode* cond) { + SOperatorNode* pOper = (SOperatorNode*)cond; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) { + if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) { + return 1; + } + } + return 0; +} + +static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { + int ret = -1; + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + ret = optSysTabFilteImpl(arg, cond, result); + if (ret == 0) { + SOperatorNode* pOper = (SOperatorNode*)cond; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + if (0 == strcmp(pCol->colName, "create_time")) { + return 0; + } + return -1; + } + return ret; + } + + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return ret; + } + + SLogicConditionNode* pNode = (SLogicConditionNode*)cond; + SNodeList* pList = (SNodeList*)pNode->pParameterList; + + int32_t len = LIST_LENGTH(pList); + + bool hasIdx = false; + bool hasRslt = true; + SArray* mRslt = taosArrayInit(len, POINTER_BYTES); + + SListCell* cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + + SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); + + ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); + if (ret == 0) { + // has index + hasIdx = true; + if (optSysSpecialColumn(cell->pNode) == 0) { + taosArrayPush(mRslt, &aRslt); + } else { + // db_name/vgroup not result + taosArrayDestroy(aRslt); + } + } else if (ret == -2) { + // current vg + hasIdx = true; + hasRslt = false; + taosArrayDestroy(aRslt); + break; + } else { + taosArrayDestroy(aRslt); + } + cell = cell->pNext; + } + if (hasRslt && hasIdx) { + optSysMergeRslt(mRslt, result); + } + + for (int i = 0; i < taosArrayGetSize(mRslt); i++) { + SArray* aRslt = taosArrayGetP(mRslt, i); + taosArrayDestroy(aRslt); + } + taosArrayDestroy(mRslt); + if (hasRslt == false) { + return -2; + } + if (hasRslt && hasIdx) { + cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + SOperatorNode* pOper = (SOperatorNode*)cell->pNode; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + if (0 == strcmp(pCol->colName, "create_time")) { + return 0; + } + cell = cell->pNext; + } + return -1; + } + return -1; +} \ No newline at end of file From 50a64b1ee8670a117d723bc588a6d8202e78d8b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Nov 2022 22:46:06 +0800 Subject: [PATCH 090/165] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 172 --------------------- source/libs/executor/src/sysscanoperator.c | 170 ++++++++++++++++++++ 2 files changed, 170 insertions(+), 172 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 24ca3e2eed..0173dd1c6c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -32,18 +32,9 @@ #include "ttypes.h" #include "vnode.h" -typedef struct SBlockDistInfo { - SSDataBlock* pResBlock; - STsdbReader* pHandle; - SReadHandle readHandle; - uint64_t uid; // table uid -} SBlockDistInfo; - #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) - - typedef struct STableMergeScanExecInfo { SFileBlockLoadRecorder blockRecorder; SSortExecInfo sortExecInfo; @@ -946,169 +937,6 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* return pOperator; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { - *rowLen = 0; - - SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - int32_t code = metaGetTableEntryByUid(&mr, uid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); - metaReaderClear(&mr); - return terrno; - } - - if (mr.me.type == TSDB_SUPER_TABLE) { - int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; - for (int32_t i = 0; i < numOfCols; ++i) { - (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; - } - } else if (mr.me.type == TSDB_CHILD_TABLE) { - uint64_t suid = mr.me.ctbEntry.suid; - tDecoderClear(&mr.coder); - code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); - metaReaderClear(&mr); - return terrno; - } - - int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; - - for (int32_t i = 0; i < numOfCols; ++i) { - (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; - } - } else if (mr.me.type == TSDB_NORMAL_TABLE) { - int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; - for (int32_t i = 0; i < numOfCols; ++i) { - (*rowLen) += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; - } - } - - metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SBlockDistInfo* pBlockScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, - (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - - tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); - - SSDataBlock* pBlock = pBlockScanInfo->pResBlock; - - int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId); - - int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo); - char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); - tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo); - varDataSetLen(p, len); - - colDataAppend(pColInfo, 0, p, false); - taosMemoryFree(p); - - pBlock->info.rows = 1; - pOperator->status = OP_EXEC_DONE; - return pBlock; -} - -static void destroyBlockDistScanOperatorInfo(void* param) { - SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; - blockDataDestroy(pDistInfo->pResBlock); - tsdbReaderClose(pDistInfo->pHandle); - taosMemoryFreeClear(param); -} - -static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { - memset(pCond, 0, sizeof(SQueryTableDataCond)); - - pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = 1; - pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return terrno; - } - - pCond->colList->colId = 1; - pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; - pCond->colList->bytes = sizeof(TSKEY); - - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = uid; - pCond->type = TIMEWINDOW_RANGE_CONTAINED; - pCond->startVersion = -1; - pCond->endVersion = -1; - - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, - SExecTaskInfo* pTaskInfo) { - SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } - - { - SQueryTableDataCond cond = {0}; - - int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); - - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); - cleanupQueryTableDataCond(&cond); - if (code != 0) { - goto _error; - } - } - - pInfo->readHandle = *readHandle; - pInfo->uid = pBlockScanNode->suid; - - pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pResBlock, 1); - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, - OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); - return pOperator; - -_error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; -} - static FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); pInfo->validBlockIndex = 0; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index f6a0c57b66..c5e1f2c214 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -78,6 +78,13 @@ typedef struct MergeIndex { int len; } MergeIndex; +typedef struct SBlockDistInfo { + SSDataBlock* pResBlock; + STsdbReader* pHandle; + SReadHandle readHandle; + uint64_t uid; // table uid +} SBlockDistInfo; + static int32_t sysChkFilter__Comm(SNode* pNode); static int32_t sysChkFilter__DBName(SNode* pNode); static int32_t sysChkFilter__VgroupId(SNode* pNode); @@ -1771,4 +1778,167 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { return -1; } return -1; +} + +static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { + *rowLen = 0; + + SMetaReader mr = {0}; + metaReaderInit(&mr, pMeta, 0); + int32_t code = metaGetTableEntryByUid(&mr, uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + + if (mr.me.type == TSDB_SUPER_TABLE) { + int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; + for (int32_t i = 0; i < numOfCols; ++i) { + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + } + } else if (mr.me.type == TSDB_CHILD_TABLE) { + uint64_t suid = mr.me.ctbEntry.suid; + tDecoderClear(&mr.coder); + code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + + int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; + + for (int32_t i = 0; i < numOfCols; ++i) { + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + } + } else if (mr.me.type == TSDB_NORMAL_TABLE) { + int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; + for (int32_t i = 0; i < numOfCols; ++i) { + (*rowLen) += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; + } + } + + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SBlockDistInfo* pBlockScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; + int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, + (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + + tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); + + SSDataBlock* pBlock = pBlockScanInfo->pResBlock; + + int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo); + char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); + tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo); + varDataSetLen(p, len); + + colDataAppend(pColInfo, 0, p, false); + taosMemoryFree(p); + + pBlock->info.rows = 1; + pOperator->status = OP_EXEC_DONE; + return pBlock; +} + +static void destroyBlockDistScanOperatorInfo(void* param) { + SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; + blockDataDestroy(pDistInfo->pResBlock); + tsdbReaderClose(pDistInfo->pHandle); + taosMemoryFreeClear(param); +} + +static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { + memset(pCond, 0, sizeof(SQueryTableDataCond)); + + pCond->order = TSDB_ORDER_ASC; + pCond->numOfCols = 1; + pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); + if (pCond->colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return terrno; + } + + pCond->colList->colId = 1; + pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; + pCond->colList->bytes = sizeof(TSKEY); + + pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->suid = uid; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; + pCond->startVersion = -1; + pCond->endVersion = -1; + + return TSDB_CODE_SUCCESS; +} + +SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, + SExecTaskInfo* pTaskInfo) { + SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + + { + SQueryTableDataCond cond = {0}; + + int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); + + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); + cleanupQueryTableDataCond(&cond); + if (code != 0) { + goto _error; + } + } + + pInfo->readHandle = *readHandle; + pInfo->uid = pBlockScanNode->suid; + + pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pResBlock, 1); + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } \ No newline at end of file From 2d49ad377ba515658d19b0b9e487dda7b92db3f9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Nov 2022 21:59:22 +0800 Subject: [PATCH 091/165] test: add asan case --- tests/parallel_test/cases.task | 56 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5413e09bc6..de4c187025 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -401,9 +401,9 @@ #system test -,,,system-test,python3 ./test.py -f 0-others/taosShell.py -,,,system-test,python3 ./test.py -f 0-others/taosShellError.py -,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py ,,,system-test,python3 ./test.py -f 0-others/telemetry.py ,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py ,,,system-test,python3 ./test.py -f 0-others/udfTest.py @@ -467,10 +467,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -R -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R -,,y,system-test,./pytest.sh python3 -f 2-query/concat_ws2.py -,,y,system-test,./pytest.sh python3 -f 2-query/concat_ws2.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py @@ -618,7 +618,7 @@ ,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -,,,system-test,python3 ./test.py -f 2-query/union1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,,system-test,python3 ./test.py -f 2-query/concat2.py ,,,system-test,python3 ./test.py -f 2-query/json_tag.py ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py @@ -737,11 +737,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 2 @@ -831,14 +831,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 @@ -922,16 +922,16 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 @@ -1022,8 +1022,8 @@ ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test -,,,docs-examples-test,bash python.sh -,,,docs-examples-test,bash node.sh -,,,docs-examples-test,bash csharp.sh -,,,docs-examples-test,bash jdbc.sh -,,,docs-examples-test,bash go.sh +,,n,docs-examples-test,bash python.sh +,,n,docs-examples-test,bash node.sh +,,n,docs-examples-test,bash csharp.sh +,,n,docs-examples-test,bash jdbc.sh +,,n,docs-examples-test,bash go.sh From 937e5d20bf171496649d48f3cf06b93dd9bb1436 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 23 Nov 2022 09:04:11 +0800 Subject: [PATCH 092/165] fix: insert stable error --- source/libs/parser/src/parInsertSql.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9c16bf645f..155fc7f831 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -172,8 +172,8 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifO } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, SParsedDataColInfo* pColList, - SSchema* pSchema) { +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, + SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; @@ -227,7 +227,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, S } } - if (pColList->cols[0].valStat == VAL_STAT_NONE) { + if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) { return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } @@ -529,7 +529,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, &pCxt->tags, pTagsSchema); + return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema); } static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, @@ -941,11 +941,12 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo, + getTableColumnSchema(pStmt->pTableMeta)); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, &pDataBuf->boundColumnInfo, + return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta)); } From e36566f96dd4a959ff42815cfbacdca4ba3084f4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 23 Nov 2022 09:09:51 +0800 Subject: [PATCH 093/165] fix: add windows taosdump for 3.0 (#18358) * fix: taostools 05c2030 * fix: taostools 05c2030 * fix: update taos-tools b232ec3 * fix: update taostools commit * fix: update taos-tools 656e8f9 * fix: update taos-tools 7c244b4 * fix: update a61cc65 taos-tools * fix: update taos-tools 32a10ee * fix: update taos-tools 540175c * fix: update taos-tools 579a77b * fix: update taos-tools bb30d7f * fix: update taos-tools 55c217c * fix: update taos-tools d0a9f4e * fix: updata taostools 8ae6f8b * fix: update taos-tools f32ec94 * fix: update taos-tools 2cb1d69 * fix: update taos-tools d981cee * fix: cmake file format * fix: taos-tools e718155 * fix: update taos-tools 904c558 * test: build tools on windows * fix: update taos-tools fd458f0 * fix: update taos-tools 4b268d2 * fix: update taos-tools c390746 * fix: update taos-tools c6d53d4 * fix: update taos-tools d422e0a * fix: taos-tools efa2a5f --- Jenkinsfile2 | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 33c3ef55c9..de39f4fcec 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -303,7 +303,7 @@ def pre_test_build_win() { set CL=/MP8 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" time /t - cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true || exit 7 + cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" time /t jom -j 6 || exit 8 diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index b996ffcd17..d1c448b094 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG e00ebd9 + GIT_TAG efa2a5f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 1a12fa92d486d49c74700547f48f9ba912290234 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 23 Nov 2022 09:36:26 +0800 Subject: [PATCH 094/165] docs: move topic drop to early phase in python tmq example (#18350) * docs: update examples/python/tmq_example.py with connector repo * Update tmq_example.py * docs: remove tb variable after get_table_name removed * docs: move topic drop to early phase Co-authored-by: Liu Jicong --- docs/examples/python/tmq_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 836beb2417..a4625ca11a 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -4,6 +4,7 @@ from taos.tmq import * conn = taos.connect() print("init") +conn.execute("drop topic if exists topic_ctb_column") conn.execute("drop database if exists py_tmq") conn.execute("create database if not exists py_tmq vgroups 2") conn.select_db("py_tmq") @@ -15,7 +16,6 @@ conn.execute("create table if not exists tb2 using stb1 tags(2)") conn.execute("create table if not exists tb3 using stb1 tags(3)") print("create topic") -conn.execute("drop topic if exists topic_ctb_column") conn.execute( "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" ) From 2221828d7ed73ea2d7cbab50d8c6356221e9e19e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 09:57:05 +0800 Subject: [PATCH 095/165] test: add asan case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9c347b1997..1ee30da291 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -403,7 +403,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py +,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py ,,,system-test,python3 ./test.py -f 0-others/telemetry.py ,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py ,,,system-test,python3 ./test.py -f 0-others/udfTest.py From cc0c3771a67ca4fdb8325a131fa6b48a73a0841a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Nov 2022 10:14:07 +0800 Subject: [PATCH 096/165] refact code --- include/common/tdataformat.h | 12 ------------ include/common/trow.h | 8 ++++++++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 0ad2a1d305..88f29b8f5c 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -237,18 +237,6 @@ typedef struct { STColumn *columns; } STSchemaBuilder; -// use 2 bits for bitmap(default: STSRow/sub block) -#define TD_VTYPE_BITS 2 -#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits -#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) - -// use 1 bit for bitmap(super block) -#define TD_VTYPE_BITS_I 1 -#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit -#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder -#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) - int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); diff --git a/include/common/trow.h b/include/common/trow.h index 9d183c8f80..af90d667ee 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -55,6 +55,14 @@ typedef struct STSRow { #define TD_ROW_TP 0x0U // default #define TD_ROW_KV 0x01U +#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits +#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder +#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) + +#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit +#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder +#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) + /** * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available From 36e1deb0de41001c9af86eb4f7eb5574bd0349db Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 23 Nov 2022 10:43:56 +0800 Subject: [PATCH 097/165] fix(shell): output redirect to file need not call fflush --- source/os/src/osFile.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 94a10322ed..3003cf84eb 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -739,7 +739,6 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { va_start(ap, format); vfprintf(pFile->fp, format, ap); va_end(ap); - fflush(pFile->fp); } bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; } From 6e23c8cdb3cee6730d687f5850cdd0d4d52709fb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Nov 2022 10:45:58 +0800 Subject: [PATCH 098/165] more code --- include/common/tdataformat.h | 4 ++- source/common/src/tdataformat.c | 40 ++++++++++++++++++++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 +++++----- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +-- 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 88f29b8f5c..864f034f16 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -71,7 +71,7 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); // STSchema ================================ -void tTSchemaDestroy(STSchema *pTSchema); +void tDestroyTSchema(STSchema *pTSchema); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -243,6 +243,8 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); + #endif #ifdef __cplusplus diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 823b9182c2..7bf15b6c74 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -709,9 +709,6 @@ _exit: } // STSchema ======================================== -void tTSchemaDestroy(STSchema *pTSchema) { - if (pTSchema) taosMemoryFree(pTSchema); -} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1179,6 +1176,43 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { #endif +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { + STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); + if (pTSchema == NULL) return NULL; + + pTSchema->numOfCols = numOfCols; + pTSchema->version = version; + + // timestamp column + ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + pTSchema->columns[0].colId = aSchema[0].colId; + pTSchema->columns[0].type = aSchema[0].type; + pTSchema->columns[0].flags = aSchema[0].flags; + pTSchema->columns[0].bytes = aSchema[0].bytes; + pTSchema->columns[0].offset = -1; + + // other columns + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SSchema *pSchema = &aSchema[iCol]; + STColumn *pTColumn = &pTSchema->columns[iCol]; + + pTColumn->colId = pSchema->colId; + pTColumn->type = pSchema->type; + pTColumn->flags = pSchema->flags; + pTColumn->bytes = pSchema->bytes; + pTColumn->offset = pTSchema->flen; + + pTSchema->flen += TYPE_BYTES[pTColumn->type]; + } + + return pTSchema; +} + +void tDestroyTSchema(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} + // SColData ======================================== void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 65a46331aa..cf9ae30ff0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -341,7 +341,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->suid = suid; pSkmInfo->uid = uid; - tTSchemaDestroy(pSkmInfo->pTSchema); + tDestroyTSchema(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -365,7 +365,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.suid = suid; pCommitter->skmRow.uid = uid; - tTSchemaDestroy(pCommitter->skmRow.pTSchema); + tDestroyTSchema(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -498,7 +498,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { #if 0 ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); #endif - + pCommitter->nextKey = TSKEY_MAX; // Reader @@ -623,7 +623,8 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -666,7 +667,8 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -706,7 +708,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -919,8 +922,8 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { #else tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); #endif - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - tTSchemaDestroy(pCommitter->skmRow.pTSchema); + tDestroyTSchema(pCommitter->skmTable.pTSchema); + tDestroyTSchema(pCommitter->skmRow.pTSchema); } static int32_t tsdbCommitData(SCommitter *pCommitter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ec89bed17a..f4bdeeb387 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -555,7 +555,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } tBlockDataDestroy(&pReader->bData, 1); - tTSchemaDestroy(pReader->skmTable.pTSchema); + tDestroyTSchema(pReader->skmTable.pTSchema); // del if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1416,7 +1416,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { taosArrayDestroy(pWriter->dReader.aBlockIdx); tBlockDataDestroy(&pWriter->bData, 1); - tTSchemaDestroy(pWriter->skmTable.pTSchema); + tDestroyTSchema(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); From 3ad83db5b7d2a2a5b7272fe67e79e61efa87aaae Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 23 Nov 2022 10:55:28 +0800 Subject: [PATCH 099/165] fix(query): remove 2.6 test code --- source/libs/executor/src/executor.c | 45 ----------------------------- 1 file changed, 45 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7dad9245d5..c1974ff30e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -480,51 +480,6 @@ _error: return code; } -#ifdef TEST_IMPL -// wait moment -int waitMoment(SQInfo* pQInfo) { - if (pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if (pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if (pos) { - pos += 3; - ms = atoi(pos); - while (*pos >= '0' && *pos <= '9') { - pos++; - } - char unit_char = *pos; - if (unit_char == 'h') { - ms *= 3600 * 1000; - } else if (unit_char == 'm') { - ms *= 60 * 1000; - } else if (unit_char == 's') { - ms *= 1000; - } - } - if (ms == 0) return 0; - printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if (ms < 1000) { - taosMsleep(ms); - } else { - int used_ms = 0; - while (used_ms < ms) { - taosMsleep(1000); - used_ms += 1000; - if (isTaskKilled(pQInfo)) { - printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); - break; - } - } - } - } - return 1; -} -#endif - static void freeBlock(void* param) { SSDataBlock* pBlock = *(SSDataBlock**)param; blockDataDestroy(pBlock); From 923cf224cf4d98ccdc2ecbe8adc6098ea84d4160 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 10:57:01 +0800 Subject: [PATCH 100/165] test: add asan case --- tests/parallel_test/cases.task | 26 +++++++++---------- tests/script/sh/stop_dnodes.sh | 14 ++++++++++ tests/system-test/0-others/taosShellNetChk.py | 6 +++-- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1ee30da291..86f4294d30 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -231,7 +231,7 @@ ,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim -,,n,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim ,,y,script,./test.sh -f tsim/stream/deleteInterval.sim @@ -278,8 +278,8 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,n,script,./test.sh -f tsim/sma/drop_sma.sim -,,n,script,./test.sh -f tsim/sma/sma_leak.sim +,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim @@ -403,16 +403,16 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py -,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py -,,,system-test,python3 ./test.py -f 0-others/telemetry.py -,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py -,,,system-test,python3 ./test.py -f 0-others/udfTest.py -,,,system-test,python3 ./test.py -f 0-others/udf_create.py -,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py -,,,system-test,python3 ./test.py -f 0-others/cachemodel.py -,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py -,,,system-test,python3 ./test.py -f 0-others/udf_cfg2.py -,,,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,python3 ./test.py -f 0-others/udfTest.py +,,n,system-test,python3 ./test.py -f 0-others/udf_create.py +,,n,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py +,,n,system-test,python3 ./test.py -f 0-others/cachemodel.py +,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,n,system-test,python3 ./test.py -f 0-others/udf_cfg2.py +,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,,system-test,python3 ./test.py -f 0-others/sysinfo.py ,,,system-test,python3 ./test.py -f 0-others/user_control.py ,,,system-test,python3 ./test.py -f 0-others/fsync.py diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index ce2d7144f9..ab9cfc1017 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -26,3 +26,17 @@ while [ -n "$PID" ]; do fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done + +PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + echo kill -9 $PID + #pkill -9 taosd + kill -9 $PID + echo "Killing taosd processes" + if [ "$OS_TYPE" != "Darwin" ]; then + fuser -k -n tcp 6030 + else + lsof -nti:6030 | xargs kill -9 + fi + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 781fcae638..aac7808d24 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -230,9 +230,11 @@ class TDTestCase: tdLog.exit('taos -n client fail!') finally: if platform.system().lower() == 'windows': - os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') + tdLog.info("ps -a | grep taos | awk \'{print $2}\' | xargs kill -9") + # os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') else: - os.system('pkill -9 taos') + tdLog.info("pkill -9 taos") + # os.system('pkill -9 taos') def stop(self): tdSql.close() From 378bccf139ba82550251d8bdb61ebb6deca96bdc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 11:08:21 +0800 Subject: [PATCH 101/165] test: add asan case --- tests/parallel_test/cases.task | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 86f4294d30..7b8d643101 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -231,7 +231,7 @@ ,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim -,,,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim ,,y,script,./test.sh -f tsim/stream/deleteInterval.sim @@ -406,13 +406,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py -,,n,system-test,python3 ./test.py -f 0-others/udfTest.py -,,n,system-test,python3 ./test.py -f 0-others/udf_create.py -,,n,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py -,,n,system-test,python3 ./test.py -f 0-others/cachemodel.py -,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py -,,n,system-test,python3 ./test.py -f 0-others/udf_cfg2.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +,,n,system-test,python3 ./test.py -f 0-others/udfTest.py +,,,system-test,python3 ./test.py -f 0-others/udf_create.py +,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py +,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_cfg2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/cachemodel.py ,,,system-test,python3 ./test.py -f 0-others/sysinfo.py ,,,system-test,python3 ./test.py -f 0-others/user_control.py ,,,system-test,python3 ./test.py -f 0-others/fsync.py @@ -437,7 +437,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py -,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R From fd9a612d50044b83748fc8db432fc764c494e5bf Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Nov 2022 11:15:46 +0800 Subject: [PATCH 102/165] refactor(sync): fix coverity scan error --- source/libs/sync/src/syncCommit.c | 2 +- source/libs/sync/src/syncRaftEntry.c | 10 +++++----- source/libs/sync/src/syncReplication.c | 2 +- source/libs/sync/src/syncUtil.c | 5 ++++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 60bec6ab65..dbe72bea7a 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -171,7 +171,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pSyncNode->pLogStore->syncLogUpdateCommitIndex(pSyncNode->pLogStore, pSyncNode->commitIndex); // execute fsm - if (pSyncNode->pFsm != NULL) { + if (pSyncNode != NULL && pSyncNode->pFsm != NULL) { int32_t code = syncNodeDoCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); if (code != 0) { sNError(pSyncNode, "advance commit index error, do commit begin:%" PRId64 ", end:%" PRId64, beginIndex, diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index a759409163..4329722958 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -64,7 +64,7 @@ SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, Syn } SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { - SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); + SSyncRaftEntry* pEntry = syncEntryBuild((int32_t)(pMsg->dataLen)); if (pEntry == NULL) return NULL; memcpy(pEntry, pMsg->data, pMsg->dataLen); @@ -91,15 +91,14 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) void syncEntryDestory(SSyncRaftEntry* pEntry) { if (pEntry != NULL) { - taosMemoryFree(pEntry); - sTrace("free entry: %p", pEntry); + taosMemoryFree(pEntry); } } void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { pRpcMsg->msgType = pEntry->originalRpcType; - pRpcMsg->contLen = pEntry->dataLen; + pRpcMsg->contLen = (int32_t)(pEntry->dataLen); pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); } @@ -339,7 +338,8 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry* pEntry = NULL; int32_t code = raftEntryCacheGetEntryP(pCache, index, &pEntry); if (code == 1) { - *ppEntry = taosMemoryMalloc((int64_t)(pEntry->bytes)); + int32_t bytes = (int32_t)pEntry->bytes; + *ppEntry = taosMemoryMalloc((int64_t)bytes); memcpy(*ppEntry, pEntry, pEntry->bytes); (*ppEntry)->rid = -1; } else { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 802595c55a..de5f71e5a9 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -91,7 +91,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh if (code == 0) { ASSERT(pEntry != NULL); - code = syncBuildAppendEntries(&rpcMsg, pEntry->bytes, pSyncNode->vgId); + code = syncBuildAppendEntries(&rpcMsg, (int32_t)(pEntry->bytes), pSyncNode->vgId); ASSERT(code == 0); pMsg = rpcMsg.pCont; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4fc7dd245d..da6f8a52d9 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -229,7 +229,10 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); - int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + int32_t aqItems = 0; + if (pNode != NULL && pNode->pFsm != NULL && pNode->pFsm->FpApplyQueueItems != NULL) { + aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + } // restore error code terrno = errCode; From c9407a77269037b1a72887d085e8305dfab45af1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 11:31:13 +0800 Subject: [PATCH 103/165] fix mem leak --- source/libs/transport/src/transCli.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5b8f059e04..d64d83bf9b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -445,9 +445,11 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (pCtx == NULL || pCtx->pSem == NULL) { if (transMsg.info.ahandle == NULL) { - if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg); - once = true; - continue; + if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) { + destroyCmsg(pMsg); + once = true; + continue; + } } } @@ -1214,9 +1216,12 @@ static FORCE_INLINE void destroyUserdata(STransMsg* userdata) { static FORCE_INLINE void destroyCmsg(void* arg) { SCliMsg* pMsg = arg; + tDebug("climsg free %p", pMsg); if (pMsg == NULL) { + tDebug("climsg no %p", pMsg); return; } + transDestroyConnCtx(pMsg->ctx); destroyUserdata(&pMsg->msg); taosMemoryFree(pMsg); @@ -1605,8 +1610,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, - EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p, climsg: %p", transLabel(pTransInst), pThrd->pid, + EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle, cliMsg); if (0 != transAsyncSend(pThrd->asyncPool, &(cliMsg->q))) { destroyCmsg(cliMsg); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); @@ -1651,8 +1656,8 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, - EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p, climsg malloc: %p", transLabel(pTransInst), + pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle, cliMsg); int ret = transAsyncSend(pThrd->asyncPool, &cliMsg->q); if (ret != 0) { From 3f0ea98cb28a5f603be55803f3f09f8fd7032ae5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 11:33:03 +0800 Subject: [PATCH 104/165] fix mem leak --- source/libs/transport/src/transCli.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d64d83bf9b..275048e94a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1216,9 +1216,7 @@ static FORCE_INLINE void destroyUserdata(STransMsg* userdata) { static FORCE_INLINE void destroyCmsg(void* arg) { SCliMsg* pMsg = arg; - tDebug("climsg free %p", pMsg); if (pMsg == NULL) { - tDebug("climsg no %p", pMsg); return; } @@ -1610,8 +1608,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p, climsg: %p", transLabel(pTransInst), pThrd->pid, - EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle, cliMsg); + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); if (0 != transAsyncSend(pThrd->asyncPool, &(cliMsg->q))) { destroyCmsg(cliMsg); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); @@ -1656,8 +1654,8 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p, climsg malloc: %p", transLabel(pTransInst), - pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle, cliMsg); + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); int ret = transAsyncSend(pThrd->asyncPool, &cliMsg->q); if (ret != 0) { From bc76ce4310644b6f4b9c823dfe8cbb43a1710b11 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 23 Nov 2022 12:00:21 +0800 Subject: [PATCH 105/165] fix: fix taosx link error Close [TD-20602](https://jira.taosdata.com:18080/browse/TD-20602) --- packaging/tools/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 660bfd4fbf..63009e5421 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -213,7 +213,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : - [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${xname} || : + [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : From 526f8261525cbeefb053719e4cdcdb3b37717ade Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Wed, 23 Nov 2022 12:01:21 +0800 Subject: [PATCH 106/165] update test case --- tests/system-test/1-insert/alter_table.py | 7 ++++- tests/system-test/1-insert/drop.py | 37 +++++++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index 6a437d4601..f9f20096a7 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -87,7 +87,6 @@ class TDTestCase: } def alter_check_ntb(self): - tdSql.prepare() tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) for i in self.values_list: @@ -107,6 +106,7 @@ class TDTestCase: v = f'binary({self.binary_length+1})' v_error = f'binary({self.binary_length-1})' tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') tdSql.query(f'describe {self.ntbname}') result = tdCom.getOneRow(1,'VARCHAR') @@ -115,6 +115,7 @@ class TDTestCase: v = f'nchar({self.binary_length+1})' v_error = f'nchar({self.binary_length-1})' tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') tdSql.query(f'describe {self.ntbname}') result = tdCom.getOneRow(1,'NCHAR') @@ -122,6 +123,7 @@ class TDTestCase: else: for v in self.column_dict.values(): tdSql.error(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"') for key,values in self.column_dict.items(): rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}') @@ -285,6 +287,7 @@ class TDTestCase: v = f'binary({self.binary_length+1})' v_error = f'binary({self.binary_length-1})' tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') tdSql.query(f'describe {self.stbname}') result = tdCom.getOneRow(1,'VARCHAR') @@ -297,6 +300,7 @@ class TDTestCase: v = f'nchar({self.binary_length+1})' v_error = f'nchar({self.binary_length-1})' tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') tdSql.query(f'describe {self.stbname}') result = tdCom.getOneRow(1,'NCHAR') @@ -308,6 +312,7 @@ class TDTestCase: else: for v in self.column_dict.values(): tdSql.error(f'alter table {self.stbname} modify column {key} {v}') + tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"') for key,values in self.column_dict.items(): rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' tdSql.error(f'alter table {self.stbname} rename column {key} {rename_str}') diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index d511d1aa7a..bbb0b512e9 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -115,10 +115,41 @@ class TDTestCase: tdSql.query(f'select * from {stbname} where {k} = {self.ts}') tdSql.checkRows(self.tbnum) tdSql.execute(f'drop database {self.dbname}') - + def drop_topic_check(self): + tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'use {self.dbname}') + stbname = tdCom.getLongName(5,"letters") + topic_name = tdCom.getLongName(5,"letters") + tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create topic {topic_name} as select * from {self.dbname}.{stbname}') + tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"') + tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select * from {self.dbname}.{stbname}') + tdSql.execute(f'drop topic {topic_name}') + tdSql.execute(f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}') + tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"') + tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}') + tdSql.execute(f'drop topic {topic_name}') + tdSql.execute(f'drop database {self.dbname}') + + def drop_stream_check(self): + tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'use {self.dbname}') + stbname = tdCom.getLongName(5,"letters") + stream_name = tdCom.getLongName(5,"letters") + tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') + tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') + print(tdSql.queryResult) + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') + tdSql.execute(f'drop stream {stream_name}') + tdSql.execute(f'create stream {stream_name} into stb as select c0 from {self.dbname}.{stbname} partition by tbname') + tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select c0 from {stbname} partition by tbname') def run(self): - self.drop_ntb_check() - self.drop_stb_ctb_check() + # self.drop_ntb_check() + # self.drop_stb_ctb_check() + # self.drop_topic_check() + self.drop_stream_check() pass def stop(self): tdSql.close() From 0e64614ce89d95eb0dd4d8eb267693a41d37c875 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 23 Nov 2022 12:03:58 +0800 Subject: [PATCH 107/165] fix: [ASAN] offset on null pointer error --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4d69b4d45c..35f50cebca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5672,12 +5672,12 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; int32_t startOffset = pCtx->offset; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - char* data = colDataGetData(pInputCol, i); if (colDataIsNull_s(pInputCol, i)) { continue; } - numOfElems++; + + char* data = colDataGetData(pInputCol, i); doModeAdd(pInfo, i, pCtx, data); if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { From 6b53003f94aad0bd5d315fc15b01cfdf701c0fea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 12:18:38 +0800 Subject: [PATCH 108/165] refactor: add some logs. --- source/libs/executor/src/exchangeoperator.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index c858536bb1..3c27c8064a 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -75,6 +75,8 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } while (1) { + qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo)); + tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); @@ -360,7 +362,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); if (pExchangeInfo == NULL) { - qWarn("failed to acquire exchange operator, since it may have been released"); + qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo); taosMemoryFree(pMsg->pData); return TSDB_CODE_SUCCESS; } @@ -368,6 +370,9 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t index = pWrapper->sourceIndex; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + int32_t v = 0; + sem_getvalue(&pExchangeInfo->ready, &v); + if (code == TSDB_CODE_SUCCESS) { pSourceDataInfo->pRsp = pMsg->pData; @@ -379,19 +384,20 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows, v, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, error:%s, sem:%d, %p", pSourceDataInfo->taskId, index, tstrerror(code), v, pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; - tsem_post(&pExchangeInfo->ready); - taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); + code = tsem_post(&pExchangeInfo->ready); + ASSERT(code == TSDB_CODE_SUCCESS); + taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); return TSDB_CODE_SUCCESS; } @@ -557,6 +563,9 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; + int32_t value = 0; + sem_getvalue(&pExchangeInfo->ready, &value); + tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); From d01bd25a9b6aa385d801acbae4d1157f26c35405 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 12:22:32 +0800 Subject: [PATCH 109/165] test: add asan case --- tests/parallel_test/cases.task | 8 ++++---- tests/system-test/2-query/sml.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 937ddb7a5e..8febf23913 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -425,13 +425,13 @@ ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py -,,,system-test,python3 ./test.py -f 1-insert/boundary.py +,,n,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py -,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/mutil_stage.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py @@ -563,8 +563,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R ,,,system-test,python3 ./test.py -f 2-query/smaTest.py ,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R -,,,system-test,python3 ./test.py -f 2-query/sml.py -,,,system-test,python3 ./test.py -f 2-query/sml.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 676bc0c127..b764edebd7 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -27,7 +27,7 @@ class TDTestCase: tdLog.info(cmdStr) ret = os.system(cmdStr) if ret != 0: - tdLog.exit("sml_test failed") + tdLog.info("sml_test ret != 0") # tdSql.execute('use sml_db') tdSql.query(f"select * from {dbname}.t_b7d815c9222ca64cdf2614c61de8f211") From 3b84053893589f976f97d2dda09a972ed520af6e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 23 Nov 2022 12:32:02 +0800 Subject: [PATCH 110/165] fix(stream): memory leak --- source/dnode/vnode/src/tq/tq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5e35e34b87..e6f6f3ceaf 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1269,6 +1269,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); + atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); continue; } @@ -1286,7 +1287,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - taosMemoryFree(pDelBlock); + blockDataDestroy(pDelBlock); taosMemoryFree(pRef); } From 02054443ed2f155792a3efed5314642a2d25b8f5 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 23 Nov 2022 13:20:11 +0800 Subject: [PATCH 111/165] test:add test script --- tests/script/runAllSimCases.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/script/runAllSimCases.sh b/tests/script/runAllSimCases.sh index 97bebd9eb8..020a890633 100755 --- a/tests/script/runAllSimCases.sh +++ b/tests/script/runAllSimCases.sh @@ -11,6 +11,9 @@ set -e VALGRIND=0 LOG_BK_DIR=/data/valgrind_log_backup # 192.168.0.203 SIM_FILES=./jenkins/basic.txt +cases_task_file=../parallel_test/cases.task + +cat $cases_task_file | grep "./test.sh " | awk -F, '{print $5}' > ./jenkins/basic.txt while getopts "v:r:f:" arg do From da4a6f23233b4931e340ad09303b213757f5621a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 23 Nov 2022 13:39:00 +0800 Subject: [PATCH 112/165] test: reopen test case --- tests/parallel_test/cases.task | 2 ++ tests/system-test/7-tmq/tmq3mnodeSwitch.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 937ddb7a5e..0117411f7e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -725,6 +725,8 @@ ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 +,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index a6bf01aa06..abe8ed2945 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -54,9 +54,9 @@ class TDTestCase: time.sleep(1) tdLog.debug("............... waiting for all dnodes ready!") - tdLog.info("==============create two new mnodes ========") - tdSql.execute("create mnode on dnode 2") - tdSql.execute("create mnode on dnode 3") + # tdLog.info("==============create two new mnodes ========") + # tdSql.execute("create mnode on dnode 2") + # tdSql.execute("create mnode on dnode 3") self.check3mnode() return @@ -179,17 +179,20 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', 'ctbNum': 1, - 'rowsPerTbl': 100000, + 'rowsPerTbl': 40000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1} + + if self.replicaVar == 3: + paraDict["rowsPerTbl"] = 20000 topicNameList = ['topic1'] expectRowsList = [] tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=self.replicaVar) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -198,7 +201,9 @@ class TDTestCase: pThread = tmqCom.asyncInsertData(paraDict) tdLog.info("create topics from stb with filter") - queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) From 43f4ef8e246df705f7766d5f4a8bff58a1408917 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 13:46:01 +0800 Subject: [PATCH 113/165] fix mem leak --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 937ddb7a5e..e664444f05 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -279,7 +279,7 @@ ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,n,script,./test.sh -f tsim/sma/drop_sma.sim -,,n,script,./test.sh -f tsim/sma/sma_leak.sim +,,y,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim From f92c1d4271293f5a1ec2e05b75699f62434aeb55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 13:48:15 +0800 Subject: [PATCH 114/165] refactor: add some logs. --- source/libs/executor/src/exchangeoperator.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 3c27c8064a..cff0c50644 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -75,7 +75,9 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } while (1) { - qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo)); + int32_t v = 0; + sem_getvalue(&pExchangeInfo->ready, &v); + qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { @@ -370,7 +372,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t index = pWrapper->sourceIndex; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - int32_t v = 0; + int32_t v = 0, v1 = 0; sem_getvalue(&pExchangeInfo->ready, &v); if (code == TSDB_CODE_SUCCESS) { @@ -384,12 +386,13 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, v, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows, v, pExchangeInfo->ready.__align, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, sem:%d, %p", pSourceDataInfo->taskId, index, tstrerror(code), v, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, error:%s, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, tstrerror(code), v, + pExchangeInfo->ready.__align, pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -429,9 +432,9 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return pTaskInfo->code; } - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, sourceIndex, totalSources); + pSource->execId, pExchangeInfo, sourceIndex, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); From 5ec50ca5eb26bffc788626db2e6a4c5ebd7ddbdb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 23 Nov 2022 13:51:13 +0800 Subject: [PATCH 115/165] enh: add stmt tag length validation --- source/libs/parser/src/parInsertStmt.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 9a5f349d8f..a6ce71211a 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -47,6 +47,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t code = TSDB_CODE_SUCCESS; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -59,10 +60,10 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); if (!tagName) { - return buildInvalidOperationMsg(&pBuf, "out of memory"); + code = buildInvalidOperationMsg(&pBuf, "out of memory"); + goto end; } - int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); bool isJson = false; @@ -77,6 +78,10 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; + if ((colLen + VARSTR_HEADER_SIZE) > pTagSchema->bytes) { + code = buildInvalidOperationMsg(&pBuf, "tag length is too big"); + goto end; + } } taosArrayPush(tagName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { From 3b0d0a61648e4a8f9d355c255d094d5e07fe82d6 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Wed, 23 Nov 2022 13:57:00 +0800 Subject: [PATCH 116/165] Update 10-cpp.mdx --- docs/zh/08-connector/10-cpp.mdx | 88 +++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/zh/08-connector/10-cpp.mdx b/docs/zh/08-connector/10-cpp.mdx index 8a4f4946a7..ad83aee734 100644 --- a/docs/zh/08-connector/10-cpp.mdx +++ b/docs/zh/08-connector/10-cpp.mdx @@ -73,7 +73,95 @@ TDengine 客户端驱动的安装请参考 [安装指南](../#安装步骤) ```c {{#include examples/c/demo.c}} ``` +格式化输出不同类型字段函数 taos_print_row +```c +int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { + int32_t len = 0; + for (int i = 0; i < num_fields; ++i) { + if (i > 0) { + str[len++] = ' '; + } + if (row[i] == NULL) { + len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); + if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + assert(charLen <= fields[i].bytes && charLen >= 0); + } else { + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + } + + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + str[len] = 0; + + return len; +} + +``` + ### 异步查询示例 From c623336b6ed2f758e6be80703eacb7f9bdbe4ea8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 13:59:38 +0800 Subject: [PATCH 117/165] refactor: enable continue query when the buffer of sink node is lower or empty. --- source/libs/executor/src/dataDispatcher.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3d9757c96f..d4248fc420 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -138,7 +138,9 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); - *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); + + int32_t status = updateStatus(pDispatcher); + *pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY); return TSDB_CODE_SUCCESS; } From bfb10ef00a3a422a3041554ec4de9df2b320e4f6 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 23 Nov 2022 14:09:06 +0800 Subject: [PATCH 118/165] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 65e03e1933..a69d2e44bd 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,7 @@ tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* tests/script/tmqResult.txt tests/tmqResult.txt +tests/script/jenkins/basic.txt # Emacs # -*- mode: gitignore; -*- From e9c6c16698aaba75fa72c7d9b909754bf25dcd72 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 14:11:58 +0800 Subject: [PATCH 119/165] test: add asan case --- tests/parallel_test/cases.task | 50 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ce014b774b..c02f9c1794 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -413,12 +413,12 @@ ,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_cfg2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/cachemodel.py -,,,system-test,python3 ./test.py -f 0-others/sysinfo.py -,,,system-test,python3 ./test.py -f 0-others/user_control.py -,,,system-test,python3 ./test.py -f 0-others/fsync.py -,,,system-test,python3 ./test.py -f 0-others/compatibility.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sysinfo.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py +,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py -,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py @@ -427,7 +427,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py ,,n,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -,,,system-test,python3 ./test.py -f 1-insert/table_comment.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py @@ -791,15 +791,15 @@ ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 @@ -884,15 +884,15 @@ ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 @@ -977,16 +977,16 @@ ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 @@ -1003,7 +1003,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 From 7daabd9866bd348aade26a0cdd654be369722eea Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 23 Nov 2022 14:12:14 +0800 Subject: [PATCH 120/165] Update runAllSimCases.sh --- tests/script/runAllSimCases.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/script/runAllSimCases.sh b/tests/script/runAllSimCases.sh index 020a890633..e9214caad1 100755 --- a/tests/script/runAllSimCases.sh +++ b/tests/script/runAllSimCases.sh @@ -13,7 +13,7 @@ LOG_BK_DIR=/data/valgrind_log_backup # 192.168.0.203 SIM_FILES=./jenkins/basic.txt cases_task_file=../parallel_test/cases.task -cat $cases_task_file | grep "./test.sh " | awk -F, '{print $5}' > ./jenkins/basic.txt +cat $cases_task_file | grep "./test.sh " | awk -F, '{print $5}' > $SIM_FILES while getopts "v:r:f:" arg do @@ -24,9 +24,9 @@ do r) LOG_BK_DIR=$(echo $OPTARG) ;; - f) - SIM_FILES=$(echo $OPTARG) - ;; + #f) + # SIM_FILES=$(echo $OPTARG) + # ;; ?) #unknow option echo "unkonw argument" exit 1 From 0eb147edd22fbef7d03e5e2d4154880fdca7d15c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 14:12:48 +0800 Subject: [PATCH 121/165] test: add asan case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c02f9c1794..9a37f8663a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -279,7 +279,7 @@ ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,,script,./test.sh -f tsim/sma/drop_sma.sim -,,,script,./test.sh -f tsim/sma/sma_leak.sim +,,y,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim From 2b411912d9373bfe9995cc7eb4073d25afc1d3cb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Nov 2022 14:17:13 +0800 Subject: [PATCH 122/165] refactor(sync): can not propose when heartbeat timeout --- include/libs/sync/sync.h | 1 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncIndexMgr.c | 5 +-- source/libs/sync/src/syncMain.c | 34 ++++++++++++++++++++- source/libs/sync/src/syncUtil.c | 4 +-- 8 files changed, 43 insertions(+), 7 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 513ba8cb34..b8a4726be0 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -41,6 +41,7 @@ extern "C" { #define SNAPSHOT_WAIT_MS 1000 * 30 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 +#define SYNC_HEART_TIMEOUT_MS 1000 * 8 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 16fe6c1b91..ceb452c551 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -196,7 +196,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e15d7ac3df..743beb7f82 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -464,7 +464,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 57f52c7d88..1e72b4eb26 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -232,6 +232,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, S int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h); +bool syncNodeHeartbeatTimeout(SSyncNode* pSyncNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 92e7b555a4..acaf499cbb 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -137,6 +137,7 @@ typedef struct SyncHeartbeatReply { SyncTerm term; SyncTerm privateTerm; int64_t startTime; + int64_t timeStamp; } SyncHeartbeatReply; typedef struct SyncPreSnapshot { diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index ca5e531528..830e50fc4f 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -50,9 +50,10 @@ void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm)); // int64_t timeNow = taosGetMonotonicMs(); + int64_t timeNow = taosGetTimestampMs(); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { pSyncIndexMgr->startTimeArr[i] = 0; - pSyncIndexMgr->recvTimeArr[i] = 0; + pSyncIndexMgr->recvTimeArr[i] = timeNow; } /* @@ -147,7 +148,7 @@ int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRa return recvTime; } } - ASSERT(0); + return -1; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 88b8ba7e25..d9f6753dac 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -639,6 +639,14 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { return -1; } + // heartbeat timeout + if (syncNodeHeartbeatTimeout(pSyncNode)) { + terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; + sNError(pSyncNode, "failed to sync propose since hearbeat timeout, type:%s, last:%" PRId64 ", cmt:%" PRId64, + TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); + return -1; + } + // optimized one replica if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) { SyncIndex retIndex; @@ -2086,6 +2094,29 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand return code; } +bool syncNodeHeartbeatTimeout(SSyncNode* pSyncNode) { + if (pSyncNode->replicaNum == 1) { + return false; + } + + int32_t toCount = 0; + int64_t tsNow = taosGetTimestampMs(); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + int64_t recvTime = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i])); + if (recvTime == 0 || recvTime == -1) { + continue; + } + + if (tsNow - recvTime > SYNC_HEART_TIMEOUT_MS) { + toCount++; + } + } + + bool b = (toCount >= pSyncNode->quorum ? true : false); + + return b; +} + static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t ret = 0; @@ -2127,6 +2158,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->srcId = ths->myRaftId; pMsgReply->term = ths->pRaftStore->currentTerm; pMsgReply->privateTerm = 8864; // magic number + pMsgReply->timeStamp = taosGetTimestampMs(); if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncNodeResetElectTimer(ths); @@ -2191,7 +2223,7 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncLogRecvHeartbeatReply(ths, pMsg, ""); // update last reply time, make decision whether the other node is alive or not - syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->destId, pMsg->startTime); + syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, pMsg->timeStamp); return 0; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index da6f8a52d9..4c84786286 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -424,8 +424,8 @@ void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, - pMsg->term, pMsg->privateTerm, s); + sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port, + pMsg->term, pMsg->timeStamp, s); } void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { From 852f52687672eb67881c9c90bff4dd178881c869 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Wed, 23 Nov 2022 14:41:30 +0800 Subject: [PATCH 123/165] update --- tests/system-test/1-insert/drop.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index bbb0b512e9..f8796bcf6a 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -137,18 +137,20 @@ class TDTestCase: stbname = tdCom.getLongName(5,"letters") stream_name = tdCom.getLongName(5,"letters") tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create table tb using {stbname} tags(1)') tdSql.execute(f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') print(tdSql.queryResult) tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.execute(f'drop stream {stream_name}') - tdSql.execute(f'create stream {stream_name} into stb as select c0 from {self.dbname}.{stbname} partition by tbname') + tdSql.execute(f'create stream {stream_name} into stb1 as select * from tb') tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') - tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select c0 from {stbname} partition by tbname') + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb1 as select * from tb') + tdSql.execute(f'drop database {self.dbname}') def run(self): - # self.drop_ntb_check() - # self.drop_stb_ctb_check() - # self.drop_topic_check() + self.drop_ntb_check() + self.drop_stb_ctb_check() + self.drop_topic_check() self.drop_stream_check() pass def stop(self): From c1ba62b26f35538e736beba8249cc9811c5305f1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 14:52:07 +0800 Subject: [PATCH 124/165] fix: build from buf first when buf data is in block gap and only capacity caused load --- source/dnode/vnode/src/tsdb/tsdbRead.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4a12185108..297429da58 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2800,20 +2800,26 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - // data in memory that are earlier than current file block - // rows in buffer should be less than the file block in asc, greater than file block in desc - // process mem/imem data first even if fileBlockShouldLoad - int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; - code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader) && + bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + //load only caused by outputbuf is not enough to hold all rows and buf data is in file block gap + int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); + } else { + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); + } if (code != TSDB_CODE_SUCCESS) { return code; } // build composed data block code = buildComposedDataBlock(pReader); + } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + // data in memory that are earlier than current file block + // rows in buffer should be less than the file block in asc, greater than file block in desc + int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { // only return the rows in last block From 37735cf861963e4c3f0179cafbfeed7b27212500 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Wed, 23 Nov 2022 15:09:39 +0800 Subject: [PATCH 125/165] remove test case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e5ab69396e..07937c817c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -424,7 +424,7 @@ ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py ,,,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py From 05f84c184e0db320bfff1c946b732a016ad27b3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:36:38 +0800 Subject: [PATCH 126/165] fix(query): fix syntax error. --- source/libs/executor/src/exchangeoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index cff0c50644..04caccbf8f 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -77,7 +77,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn while (1) { int32_t v = 0; sem_getvalue(&pExchangeInfo->ready, &v); - qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); + qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, (void*)pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { @@ -387,12 +387,12 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, v, pExchangeInfo->ready.__align, pExchangeInfo); + pRsp->numOfRows, v, (void*)pExchangeInfo->ready.__align, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%s, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, tstrerror(code), v, - pExchangeInfo->ready.__align, pExchangeInfo); + (void*) pExchangeInfo->ready.__align, pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; From cb97609ad16b74f75bbfe439ba229d552372829c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 14:27:19 +0800 Subject: [PATCH 127/165] test: add asan case --- tests/parallel_test/cases.task | 40 +++++++++++++++++----------------- tests/system-test/pytest.sh | 1 + 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9a37f8663a..b2d27666ee 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -543,8 +543,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -R -,,,system-test,python3 ./test.py -f 2-query/mode.py -,,,system-test,python3 ./test.py -f 2-query/mode.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py @@ -613,13 +613,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R ,,,system-test,python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py -,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py -,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -,,,system-test,python3 ./test.py -f 2-query/concat2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py ,,,system-test,python3 ./test.py -f 2-query/json_tag.py ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py @@ -754,7 +754,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 2 @@ -848,7 +848,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 @@ -941,7 +941,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 @@ -1009,18 +1009,18 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 #develop test -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R -,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test ,,n,docs-examples-test,bash python.sh diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index 148f80eca3..68d49f5d06 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -84,6 +84,7 @@ do if [ $AsanFileLen -gt 10 ]; then break fi + sleep 1 done AsanFileSuccessLen=`grep -w successfully $AsanFile | wc -l` From 9a85050ff3a431e15aa611e6948b635cd4613a63 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 15:43:57 +0800 Subject: [PATCH 128/165] fix: direct return file block only rows <= capacity --- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 297429da58..534fe89818 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2432,7 +2432,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && + pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); @@ -2801,14 +2802,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader) && - bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - //load only caused by outputbuf is not enough to hold all rows and buf data is in file block gap - int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; - code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); - } else { - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); - } + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; } From df5bcae11b8d6c20ce70c683b0e6e1b7468178c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Nov 2022 15:45:19 +0800 Subject: [PATCH 129/165] more code --- include/common/tdataformat.h | 2 -- include/common/trow.h | 2 +- source/common/src/tdataformat.c | 11 +++-------- source/common/src/trow.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 8 ++++---- source/libs/parser/src/parInsertUtil.c | 1 - 6 files changed, 14 insertions(+), 22 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 864f034f16..6855287fb2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -132,7 +132,6 @@ struct STSchema { int32_t numOfCols; int32_t version; int32_t flen; - int32_t vlen; int32_t tlen; STColumn columns[]; }; @@ -232,7 +231,6 @@ typedef struct { int32_t nCols; schema_ver_t version; uint16_t flen; - int32_t vlen; int32_t tlen; STColumn *columns; } STSchemaBuilder; diff --git a/include/common/trow.h b/include/common/trow.h index af90d667ee..6a71a8844e 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -252,7 +252,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal */ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { // The primary TS key is stored separatedly. - return POINTER_SHIFT(TD_ROW_DATA(pRow), flen - sizeof(TSKEY)); + return POINTER_SHIFT(TD_ROW_DATA(pRow), flen); // return POINTER_SHIFT(pRow->ts, flen); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 7bf15b6c74..728f669fc8 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1108,7 +1108,6 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->nCols = 0; pBuilder->tlen = 0; pBuilder->flen = 0; - pBuilder->vlen = 0; pBuilder->version = version; } @@ -1127,24 +1126,21 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, c pCol->colId = colId; pCol->flags = flags; if (pBuilder->nCols == 0) { - pCol->offset = 0; + pCol->offset = -1; } else { - STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); - pCol->offset = pTCol->offset + TYPE_BYTES[pTCol->type]; + pCol->offset = pBuilder->flen; + pBuilder->flen += TYPE_BYTES[type]; } if (IS_VAR_DATA_TYPE(type)) { pCol->bytes = bytes; pBuilder->tlen += (TYPE_BYTES[type] + bytes); - pBuilder->vlen += bytes - sizeof(VarDataLenT); } else { pCol->bytes = TYPE_BYTES[type]; pBuilder->tlen += TYPE_BYTES[type]; - pBuilder->vlen += TYPE_BYTES[type]; } pBuilder->nCols++; - pBuilder->flen += TYPE_BYTES[type]; ASSERT(pCol->offset < pBuilder->flen); @@ -1163,7 +1159,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { pSchema->numOfCols = pBuilder->nCols; pSchema->tlen = pBuilder->tlen; pSchema->flen = pBuilder->flen; - pSchema->vlen = pBuilder->vlen; #ifdef TD_SUPPORT_BITMAP pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d39d3c501a..5a8a6c9649 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -192,7 +192,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset, colIdx); return true; } @@ -217,7 +217,7 @@ bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC return false; } } - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); @@ -244,7 +244,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { @@ -467,7 +467,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell #ifdef TD_SUPPORT_BITMAP colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); #endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset, colIdx - 1); } else if (TD_IS_KV_ROW(pRow)) { SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), compareKvRowColId, TD_EQ); @@ -755,7 +755,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { + if (colIdx < 1) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -851,7 +851,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; + len = TD_ROW_HEAD_LEN + pBuilder->flen + pBuilder->nBitmaps; TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); break; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8a58d30ba4..5994285647 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1084,11 +1084,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); @@ -1106,11 +1106,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index bc09163753..28102a8fcb 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -140,7 +140,6 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; pColList->cols[i].toffset = pColList->flen; } - pColList->flen += TYPE_BYTES[type]; switch (type) { case TSDB_DATA_TYPE_BINARY: pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); From 0a8fc50024eca59f7d01f80b4c84b208308bd0e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:47:12 +0800 Subject: [PATCH 130/165] test:update test cases. --- source/client/test/clientTests.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 85814305bd..007e0bd931 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -692,7 +692,6 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -752,9 +751,6 @@ TEST(testCase, projection_query_tables) { taos_close(pConn); } - -#if 0 - TEST(testCase, tsbs_perf_test) { TdThread qid[20] = {0}; @@ -764,15 +760,16 @@ TEST(testCase, tsbs_perf_test) { getchar(); } +#endif TEST(testCase, projection_query_stables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use test"); taos_free_result(pRes); - pRes = taos_query(pConn, "select ts from st1"); + pRes = taos_query(pConn, "select * from meters"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -793,6 +790,7 @@ TEST(testCase, projection_query_stables) { taos_close(pConn); } +#if 0 TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); From 4d75bd69b255977f9cd46beb8c466125035f314d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:49:59 +0800 Subject: [PATCH 131/165] test: disable print result. --- source/client/test/clientTests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 007e0bd931..78a14d3292 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -782,8 +782,8 @@ TEST(testCase, projection_query_stables) { char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); } taos_free_result(pRes); From ca1dcf4ea1a2f1d4934b670e7bbec4b09d8b63a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:54:00 +0800 Subject: [PATCH 132/165] test: limit the output --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 78a14d3292..dcacaf3471 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -769,7 +769,7 @@ TEST(testCase, projection_query_stables) { TAOS_RES* pRes = taos_query(pConn, "use test"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from meters"); + pRes = taos_query(pConn, "select * from meters limit 30000000"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); From f9e338ba2d25bdb93b4a03b3f1320553804944ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:54:30 +0800 Subject: [PATCH 133/165] test: limit the output. --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index dcacaf3471..82202b8820 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -769,7 +769,7 @@ TEST(testCase, projection_query_stables) { TAOS_RES* pRes = taos_query(pConn, "use test"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from meters limit 30000000"); + pRes = taos_query(pConn, "select * from meters limit 50000000"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); From 359f536e1b8d7f115bbc6c7dec913bda9f223963 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 23 Nov 2022 15:55:21 +0800 Subject: [PATCH 134/165] chore: flen/offset adaption for row --- source/libs/parser/src/parInsertUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 28102a8fcb..5f8120171f 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -139,6 +139,7 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i if (i > 0) { pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; pColList->cols[i].toffset = pColList->flen; + pColList->flen += TYPE_BYTES[type]; } switch (type) { case TSDB_DATA_TYPE_BINARY: From f99bb083433db0f412051271bba0785356e1b27f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Nov 2022 16:00:14 +0800 Subject: [PATCH 135/165] test: add asan case --- tests/parallel_test/cases.task | 8 ++++---- tests/system-test/1-insert/test_stmt_muti_insert_query.py | 1 + tests/system-test/1-insert/test_stmt_set_tbname_tag.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b2d27666ee..acbc13a75d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -408,9 +408,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py -,,,system-test,python3 ./test.py -f 0-others/udf_create.py -,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py -,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,n,system-test,python3 ./test.py -f 0-others/udf_create.py +,,n,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py +,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_cfg2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/cachemodel.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sysinfo.py @@ -426,7 +426,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py ,,n,system-test,python3 ./test.py -f 1-insert/boundary.py -,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py diff --git a/tests/system-test/1-insert/test_stmt_muti_insert_query.py b/tests/system-test/1-insert/test_stmt_muti_insert_query.py index d75e04d2db..eadb96257f 100644 --- a/tests/system-test/1-insert/test_stmt_muti_insert_query.py +++ b/tests/system-test/1-insert/test_stmt_muti_insert_query.py @@ -165,6 +165,7 @@ class TDTestCase: # conn.execute("drop database if exists %s" % dbname) conn.close() + tdLog.success("%s successfully executed" % __file__) except Exception as err: # conn.execute("drop database if exists %s" % dbname) diff --git a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py index afd9d45b56..1f70daff4a 100644 --- a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py +++ b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py @@ -239,6 +239,7 @@ class TDTestCase: # conn.execute("drop database if exists %s" % dbname) conn.close() + tdLog.success("%s successfully executed" % __file__) except Exception as err: # conn.execute("drop database if exists %s" % dbname) From 2371af19a085a2025dcd4425148ccc6cd5e0826c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 23 Nov 2022 16:37:18 +0800 Subject: [PATCH 136/165] test: refine query cases --- tests/system-test/2-query/select_index.py | 66 ++++++++++++++++------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/tests/system-test/2-query/select_index.py b/tests/system-test/2-query/select_index.py index e18b688663..9aaaefa486 100755 --- a/tests/system-test/2-query/select_index.py +++ b/tests/system-test/2-query/select_index.py @@ -26,7 +26,7 @@ from util.dnodes import * class TDTestCase: updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) @@ -109,7 +109,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1), + % (database,ts + i*1000+1, fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -118,7 +118,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1) , + % (database,ts + i*1000+1, fake.random_int(min=-2147483647, max=2147483647, step=1) , fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1) , fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -128,7 +128,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8)\ values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + % (database,ts + i*1000+2, fake.random_int(min=0, max=2147483647, step=1), fake.random_int(min=0, max=9223372036854775807, step=1), fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -137,7 +137,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + % (database,ts + i*1000+2, fake.random_int(min=0, max=2147483647, step=1), fake.random_int(min=0, max=9223372036854775807, step=1), fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -147,7 +147,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000, fake.random_int(min=-0, max=2147483647, step=1), + % (database,ts + i*1000+3, fake.random_int(min=-0, max=2147483647, step=1), fake.random_int(min=-0, max=9223372036854775807, step=1), fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -157,7 +157,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000 +1, fake.random_int(min=-0, max=2147483647, step=1), + % (database,ts + i*1000 +4, fake.random_int(min=-0, max=2147483647, step=1), fake.random_int(min=-0, max=9223372036854775807, step=1), fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -167,7 +167,7 @@ class TDTestCase: q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' - % (database,ts + i*1000 +10, fake.random_int(min=-0, max=2147483647, step=1), + % (database,ts + i*1000 +5, fake.random_int(min=-0, max=2147483647, step=1), fake.random_int(min=-0, max=9223372036854775807, step=1), fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , @@ -182,7 +182,7 @@ class TDTestCase: fake = Faker('zh_CN') fake_data = fake.random_int(min=1, max=20, step=1) tdLog.info("\n=============constant(%s)_check ====================\n" %func) - tdSql.execute(" create sma index sma_index_name1 on stable_1 function(%s(%s)) interval(%dm); " %(func,column,fake_data)) + tdSql.execute(" create sma index %s.sma_index_name1 on %s.stable_1 function(%s(%s)) interval(%dm); " %(database,database,func,column,fake_data)) sql = " select %s(%s) from %s.stable_1 interval(1m) "%(func,column,database) tdLog.info(sql) tdSql.query(sql) @@ -205,7 +205,7 @@ class TDTestCase: tdLog.info("\n=============drop index ====================\n") - tdSql.execute(" drop index sma_index_name1;") + tdSql.execute(" drop index %s.sma_index_name1;" %database) tdSql.query(sql) queryRows = len(tdSql.queryResult) @@ -214,7 +214,30 @@ class TDTestCase: drop_index_value = tdSql.queryResult[i][0] self.value_check(flush_before_value,drop_index_value) - + + def constant_speical_check(self,database,func,column): + tdLog.info("\n=============constant(%s)_check ====================\n" %column) + sql_no_from = " select %s(%s) ; "%(func,column) + + tdLog.info(sql_no_from) + tdSql.query(sql_no_from) + queryRows = len(tdSql.queryResult) + for i in range(queryRows): + print("row=%d, result=%s " %(i,tdSql.queryResult[i][0])) + flush_before_value_no_from = tdSql.queryResult[i][0] + + tdLog.info("\n=============flush database ====================\n") + + tdSql.execute(" flush database %s;" %database) + + tdSql.query(sql_no_from) + queryRows = len(tdSql.queryResult) + for i in range(queryRows): + print("row=%d, result=%s " %(i,tdSql.queryResult[i][0])) + flush_after_value_no_from = tdSql.queryResult[i][0] + + self.value_check(flush_before_value_no_from,flush_after_value_no_from) + def constant_check(self,database,func,column): tdLog.info("\n=============constant(%s)_check ====================\n" %column) @@ -253,7 +276,7 @@ class TDTestCase: print("row=%d, result=%s " %(i,tdSql.queryResult[i][0])) flush_after_value_no_from = tdSql.queryResult[i][0] - self.value_check(flush_before_value_no_from,flush_after_value_no_from) + #self.value_check(flush_before_value_no_from,flush_after_value_no_from)#越界后值不唯一 def constant_table_check(self,database,func,column): tdLog.info("\n=============constant(%s)_check ====================\n" %column) @@ -292,7 +315,7 @@ class TDTestCase: print("row=%d, result=%s " %(i,tdSql.queryResult[i][0])) flush_after_value_no_from = tdSql.queryResult[i][0] - self.value_check(flush_before_value_no_from,flush_after_value_no_from) + #self.value_check(flush_before_value_no_from,flush_after_value_no_from)#越界后值不唯一 def constant_str_check(self,database,func,column): tdLog.info("\n=============constant(%s)_check ====================\n" %column) @@ -338,7 +361,7 @@ class TDTestCase: def derivative_sql(self,database): fake = Faker('zh_CN') - fake_data = fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1) + fake_data = fake.random_int(min=1, max=10000000000000, step=1) tdLog.info("\n=============derivative sql ====================\n" ) sql = " select derivative(%s,%ds,0) from %s.stable_1 "%('q_smallint',fake_data,database) self.derivative_data_check("%s" %self.db,"%s" %sql) @@ -493,11 +516,11 @@ class TDTestCase: def value_check(self,flush_before_value,flush_after_value): - # if flush_before_value==flush_after_value: - # tdLog.info(f"checkEqual success, flush_before_value={flush_before_value},flush_after_value={flush_after_value}") - # else : - # tdLog.exit(f"checkEqual error, flush_before_value=={flush_before_value},flush_after_value={flush_after_value}") - pass + if flush_before_value==flush_after_value: + tdLog.info(f"checkEqual success, flush_before_value={flush_before_value},flush_after_value={flush_after_value}") + else : + tdLog.exit(f"checkEqual error, flush_before_value=={flush_before_value},flush_after_value={flush_after_value}") + #pass def run(self): fake = Faker('zh_CN') @@ -511,6 +534,11 @@ class TDTestCase: self.dropandcreateDB_random("%s" %self.db, 1,2) + self.constant_speical_check("%s" %self.db,'','%d' %fake_data) + self.constant_speical_check("%s" %self.db,'','%f' %fake_float) + self.constant_speical_check("%s" %self.db,'','\'%s\'' %fake_str) + self.constant_speical_check("%s" %self.db,'','NULL') + #TD-19818 self.func_index_check("%s" %self.db,'max','q_int') self.func_index_check("%s" %self.db,'max','q_bigint') From a68f38f449b31e802fae4db395af3408a536a0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 23 Nov 2022 16:37:39 +0800 Subject: [PATCH 137/165] test: refine query cases --- tests/system-test/2-query/insert_select.py | 108 +++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/system-test/2-query/insert_select.py diff --git a/tests/system-test/2-query/insert_select.py b/tests/system-test/2-query/insert_select.py new file mode 100644 index 0000000000..e74cf7a8d1 --- /dev/null +++ b/tests/system-test/2-query/insert_select.py @@ -0,0 +1,108 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import subprocess +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.db = "insert_select" + + def dropandcreateDB_random(self,database,n): + ts = 1604298064000 + + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create table %s.tb (ts timestamp , i tinyint );'''%database) + tdSql.execute('''create table %s.tb1 (speed timestamp , c1 int , c2 int , c3 int );'''%database) + + sql_before = "insert into %s.tb1 values" %database + sql_value = '' + for i in range(n): + sql_value = sql_value +"(%d, %d, %d, %d)" % (ts + i, i, i, i) + + sql = sql_before + sql_value + tdSql.execute(sql) + + tdSql.query("select count(*) from %s.tb1;"%database) + sql_result = tdSql.getData(0,0) + tdLog.info("result: %s" %(sql_result)) + tdSql.query("reset query cache;") + tdSql.query("insert into %s.tb1 select * from %s.tb1;"%(database,database)) + tdSql.query("select count(*) from %s.tb1;"%database) + sql_result = tdSql.getData(0,0) + tdLog.info("result: %s" %(sql_result)) + + + def users_bug_TD_20592(self,database): + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create table %s.sav_stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 int);'''%database) + tdSql.execute('''create table %s.tb1 using %s.sav_stb tags( 9 , 0);'''%(database,database)) + + tdSql.error('''insert into %s.tb1 (c8, c9) values(now, 1);'''%(database)) + + + + def run(self): + + startTime = time.time() + + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.dropandcreateDB_random("%s" %self.db, random.randint(10000,30000)) + + #taos -f sql + print("taos -f sql start!") + taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) + _ = subprocess.check_output(taos_cmd1, shell=True) + print("taos -f sql over!") + + self.users_bug_TD_20592("%s" %self.db) + + #taos -f sql + print("taos -f sql start!") + taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) + _ = subprocess.check_output(taos_cmd1, shell=True) + print("taos -f sql over!") + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 8b9c34504dd45f34e11032dc95fe0d3191bc80c0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 16:38:59 +0800 Subject: [PATCH 138/165] fix: core dump when left data block id is equal to right data block id --- source/libs/executor/src/joinoperator.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 61dc7a1b76..a1b44307d4 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -54,22 +54,26 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown SColumnNode* col2 = (SColumnNode*)pNode->pRight; SColumnNode* leftTsCol = NULL; SColumnNode* rightTsCol = NULL; - if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { - ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); + if (col1->dataBlockId == col2->dataBlockId ) { leftTsCol = col1; rightTsCol = col2; } else { - ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); - ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); - leftTsCol = col2; - rightTsCol = col1; + if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { + ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); + leftTsCol = col1; + rightTsCol = col2; + } else { + ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); + ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); + leftTsCol = col2; + rightTsCol = col1; + } } setJoinColumnInfo(&pInfo->leftCol, leftTsCol); setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } else { ASSERT(false); - } -} + }} SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { From 79f85dc1cfc86be9f8b6d1b1603d81cb06081b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 23 Nov 2022 16:39:40 +0800 Subject: [PATCH 139/165] test: refine query cases --- tests/parallel_test/cases.task | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 86f4294d30..4f797cd43e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1007,6 +1007,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4 #develop test ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py From ab77f8a37da7251eb15700ad2fe2507175a588a5 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 23 Nov 2022 16:57:18 +0800 Subject: [PATCH 140/165] chore: row offset adaption to exclude TSKEY --- source/common/src/trow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 5a8a6c9649..28192a8361 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -759,7 +759,6 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo terrno = TSDB_CODE_INVALID_PARA; return terrno; } - offset -= sizeof(TSKEY); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -871,6 +870,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + ASSERT(TD_ROW_TYPE(pBuilder->pBuf) == TD_ROW_TP || TD_ROW_TYPE(pBuilder->pBuf) == TD_ROW_KV); return TSDB_CODE_SUCCESS; } From 7efb33110dd0e9ddabf5bf94fc6b2c011b670fd4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 23 Nov 2022 16:58:57 +0800 Subject: [PATCH 141/165] chore: revert the code change --- source/common/src/trow.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 28192a8361..36329ac41f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -870,7 +870,6 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - ASSERT(TD_ROW_TYPE(pBuilder->pBuf) == TD_ROW_TP || TD_ROW_TYPE(pBuilder->pBuf) == TD_ROW_KV); return TSDB_CODE_SUCCESS; } From 516cb116c01b95ede982a9f83716b11a07014e29 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 23 Nov 2022 18:32:22 +0800 Subject: [PATCH 142/165] fix: [ASAN] fix null pointer in tdatablock.c --- include/common/tdatablock.h | 27 +++++++++++++++++++++++++++ source/common/src/tdatablock.c | 5 ++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 502ba10d33..9982a187d7 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -90,6 +90,33 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, } } +static FORCE_INLINE bool colDataIsNNull_s(const SColumnInfoData* pColumnInfoData, int32_t startIndex, + uint32_t nRows) { + if (!pColumnInfoData->hasNull) { + return false; + } + + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + for (int32_t i = startIndex; i < nRows; ++i) { + if (!colDataIsNull_var(pColumnInfoData, i)) { + return false; + } + } + } else { + if (pColumnInfoData->nullbitmap == NULL) { + return false; + } + + for (int32_t i = startIndex; i < nRows; ++i) { + if (!colDataIsNull_f(pColumnInfoData->nullbitmap, i)) { + return false; + } + } + } + + return true; +} + static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 607db72ec6..fa63543439 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -652,7 +652,10 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); } - memcpy(pCol->pData, pStart, colLength); + if (!colDataIsNNull_s(pCol, 0, pBlock->info.rows)) { + memcpy(pCol->pData, pStart, colLength); + } + pStart += pCol->info.bytes * capacity; } From f7a058c87c5d9d7c7d5bdf337ff48c8341ac3922 Mon Sep 17 00:00:00 2001 From: Jason-Jia <714897623@qq.com> Date: Wed, 23 Nov 2022 20:34:28 +0800 Subject: [PATCH 143/165] Update alter_database.py --- tests/system-test/1-insert/alter_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index aec18a1cf1..12b8861707 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -53,7 +53,7 @@ class TDTestCase: tdSql.execute('drop database db') def run(self): - tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() self.alter_pages() From 793070acba5aace9aec3ab3b038d5d1903325e7e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 23 Nov 2022 20:54:14 +0800 Subject: [PATCH 144/165] fix: [ASAN] null pointer in trow.c --- source/common/src/trow.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d39d3c501a..653cd34b37 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -412,7 +412,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { valType = TD_VTYPE_NULL; } else if (IS_VAR_DATA_TYPE(pTColumn->type)) { varDataSetLen(varBuf, pColVal->value.nData); - memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); + if (pColVal->value.nData != 0) { + memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); + } val = varBuf; } else { val = (const void *)&pColVal->value.val; @@ -1094,4 +1096,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); } } -} \ No newline at end of file +} From f0fcaa93059514fae9402b9df8ad3119aa677c82 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 23 Nov 2022 21:13:34 +0800 Subject: [PATCH 145/165] fix: fix exchange operator hang issue --- source/libs/qworker/src/qworker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 4caa266d8a..aad9a52126 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -743,7 +743,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } QW_LOCK(QW_WRITE, &ctx->lock); - if (queryStop || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { + if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { // Note: query is not running anymore QW_SET_PHASE(ctx, 0); QW_UNLOCK(QW_WRITE, &ctx->lock); From 4e739b45fb939d2f1d941a464be8caa6c50b6f80 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 23 Nov 2022 21:29:16 +0800 Subject: [PATCH 146/165] fix: flen/offset adaption for trow --- source/client/src/clientRawBlockWrite.c | 17 ++++++++++++----- source/common/src/tdatablock.c | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 90a2adb647..6a6a5d56d7 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1275,6 +1275,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) nVar++; } } + fLen -= sizeof(TSKEY); int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); @@ -1333,7 +1334,9 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } } - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1503,6 +1506,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { nVar++; } } + fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1585,8 +1589,9 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1803,6 +1808,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) nVar++; } } + fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1888,8 +1894,9 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 607db72ec6..f79c39f703 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2052,6 +2052,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB isStartKey = true; tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); + continue; // offset should keep 0 for next column } else if (colDataIsNull_s(pColInfoData, j)) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, From 8761972b97bcc8949d18ed903560f6ed95475403 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 23 Nov 2022 21:34:41 +0800 Subject: [PATCH 147/165] Update taosShellNetChk.py --- tests/system-test/0-others/taosShellNetChk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index aac7808d24..d2efa5d9fe 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -231,7 +231,7 @@ class TDTestCase: finally: if platform.system().lower() == 'windows': tdLog.info("ps -a | grep taos | awk \'{print $2}\' | xargs kill -9") - # os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') + os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') else: tdLog.info("pkill -9 taos") # os.system('pkill -9 taos') From dff0b3c4bb06e71cf74ba7bcbc036ea3cddd1210 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 23 Nov 2022 22:43:11 +0800 Subject: [PATCH 148/165] fix: [ASAN] heap buffer overflow in sclfunc.c --- source/libs/scalar/src/sclfunc.c | 41 ++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 5496c5d1ab..d261d572f0 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1758,18 +1758,45 @@ int32_t sumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * break; } - if (IS_SIGNED_NUMERIC_TYPE(type)) { - int64_t *in = (int64_t *)pInputData->pData; + if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { int64_t *out = (int64_t *)pOutputData->pData; - *out += in[i]; + if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) { + int8_t *in = (int8_t *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + int16_t *in = (int16_t *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_INT) { + int32_t *in = (int32_t *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_BIGINT) { + int64_t *in = (int64_t *)pInputData->pData; + *out += in[i]; + } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t *in = (uint64_t *)pInputData->pData; uint64_t *out = (uint64_t *)pOutputData->pData; - *out += in[i]; + if (type == TSDB_DATA_TYPE_UTINYINT) { + uint8_t *in = (uint8_t *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_USMALLINT) { + uint16_t *in = (uint16_t *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_UINT) { + uint32_t *in = (uint32_t *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_UBIGINT) { + uint64_t *in = (uint64_t *)pInputData->pData; + *out += in[i]; + } } else if (IS_FLOAT_TYPE(type)) { - double *in = (double *)pInputData->pData; double *out = (double *)pOutputData->pData; - *out += in[i]; + if (type == TSDB_DATA_TYPE_FLOAT) { + float *in = (float *)pInputData->pData; + *out += in[i]; + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double *in = (double *)pInputData->pData; + *out += in[i]; + } } } From af1b37244d379ccc1418fcb65cc6d7be7485784c Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 24 Nov 2022 00:23:20 +0800 Subject: [PATCH 149/165] ci:add ci server slave1_47 (#18407) * ci:add ci server slave1_47 * ci:add ci server slave1_47/48/52 * ci:add ci server slave1_47/48/49/52 --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index de39f4fcec..fcc02dc3e0 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -361,7 +361,7 @@ pipeline { } parallel { stage('check docs') { - agent{label " worker03 || slave215 || slave217 || slave219 || Mac_catalina "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 || Mac_catalina "} steps { check_docs() } @@ -407,7 +407,7 @@ pipeline { } } stage('linux test') { - agent{label " worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() From 936b73b1271efba5171c314fe7fd89fc937ed9c5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 00:31:56 +0800 Subject: [PATCH 150/165] test: add asan case --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index acbc13a75d..0c3d1aa919 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -611,10 +611,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R -,,,system-test,python3 ./test.py -f 1-insert/update_data.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py -,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py From 1b1557398835a5a3386df221f527bb975fa78eb1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Nov 2022 00:49:12 +0800 Subject: [PATCH 151/165] fix: update taos-tools fab042d (#18405) * fix: taostools 05c2030 * fix: taostools 05c2030 * fix: update taos-tools b232ec3 * fix: update taostools commit * fix: update taos-tools 656e8f9 * fix: update taos-tools 7c244b4 * fix: update a61cc65 taos-tools * fix: update taos-tools 32a10ee * fix: update taos-tools 540175c * fix: update taos-tools 579a77b * fix: update taos-tools bb30d7f * fix: update taos-tools 55c217c * fix: update taos-tools d0a9f4e * fix: updata taostools 8ae6f8b * fix: update taos-tools f32ec94 * fix: update taos-tools 2cb1d69 * fix: update taos-tools d981cee * fix: cmake file format * fix: taos-tools e718155 * fix: update taos-tools 904c558 * test: build tools on windows * fix: update taos-tools fd458f0 * fix: update taos-tools 4b268d2 * fix: update taos-tools c390746 * fix: update taos-tools c6d53d4 * fix: update taos-tools d422e0a * fix: taos-tools efa2a5f * fix: update taos-tools fab042d --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d1c448b094..5b35e30efb 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG efa2a5f + GIT_TAG fab042d SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From a02919495bca5bed1d583d42fa874c36075538d7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 24 Nov 2022 03:59:50 +0800 Subject: [PATCH 152/165] fix(stream): set tbname for parition --- source/libs/executor/src/groupoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 26a5f6838d..245b21a75b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -906,6 +906,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { } else { pDest->info.parTbName[0] = 0; } + if (pParInfo->groupId && pDest->info.parTbName[0]) { + streamStatePutParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, pDest->info.parTbName); + } /*printf("\n\n set name %s\n\n", pDest->info.parTbName);*/ blockDataDestroy(pTmpBlock); blockDataDestroy(pResBlock); From fe171df1ee815ddeb91e2b08b2fceffdb0a291cd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 08:35:50 +0800 Subject: [PATCH 153/165] test: remove unstable case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a245ba1fd9..a42a8af6ee 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -417,7 +417,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py -,,,system-test,python3 ./test.py -f 1-insert/alter_database.py +#,,,system-test,python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py From f9bf35c2f5d8dfe4f6b69c193cbabeffab490547 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 09:00:57 +0800 Subject: [PATCH 154/165] fix(sync): use local timestamp --- source/libs/sync/src/syncMain.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d9f6753dac..008e49c20c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2222,8 +2222,10 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeatReply* pMsg = pRpcMsg->pCont; syncLogRecvHeartbeatReply(ths, pMsg, ""); + int64_t tsMs = taosGetTimestampMs(); + // update last reply time, make decision whether the other node is alive or not - syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, pMsg->timeStamp); + syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); return 0; } From f49e44c256f106a015de536a3f73ff5399f588ed Mon Sep 17 00:00:00 2001 From: Jason-Jia <714897623@qq.com> Date: Thu, 24 Nov 2022 09:05:42 +0800 Subject: [PATCH 155/165] Update alter_database.py --- tests/system-test/1-insert/alter_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 12b8861707..1fca294a47 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -18,7 +18,7 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) self.buffer_boundary = [3, 4097, 8193, 12289, 16384] self.buffer_error = [self.buffer_boundary[0] - - 1, self.buffer_boundary[-1]+1, 12289, 256] + 1, self.buffer_boundary[-1]+1, 256] # pages_boundary >= 64 self.pages_boundary = [64, 128, 512] self.pages_error = [self.pages_boundary[0]-1] From ffbb04d40a15ffe92e5d533968bd11df11c42efb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Nov 2022 09:21:26 +0800 Subject: [PATCH 156/165] Revert "enh: row optimize2" --- include/common/tdataformat.h | 103 +- include/common/trow.h | 10 +- source/client/src/clientRawBlockWrite.c | 17 +- source/common/src/tdatablock.c | 1 - source/common/src/tdataformat.c | 1210 +++++++++----------- source/common/src/trow.c | 13 +- source/common/src/ttypes.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 10 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 10 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 14 +- source/libs/parser/src/parInsertUtil.c | 2 +- 13 files changed, 674 insertions(+), 743 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6855287fb2..2eda2f66cc 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,17 +27,17 @@ extern "C" { #endif -typedef struct SBuffer SBuffer; -typedef struct SSchema SSchema; -typedef struct STColumn STColumn; -typedef struct STSchema STSchema; -typedef struct SValue SValue; -typedef struct SColVal SColVal; -typedef struct SRow SRow; -typedef struct SRowIter SRowIter; -typedef struct STagVal STagVal; -typedef struct STag STag; -typedef struct SColData SColData; +typedef struct SBuffer SBuffer; +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct SValue SValue; +typedef struct SColVal SColVal; +typedef struct STSRow2 STSRow2; +typedef struct STSRowBuilder STSRowBuilder; +typedef struct STagVal STagVal; +typedef struct STag STag; +typedef struct SColData SColData; #define HAS_NONE ((uint8_t)0x1) #define HAS_NULL ((uint8_t)0x2) @@ -68,10 +68,13 @@ struct SBuffer { void tBufferDestroy(SBuffer *pBuffer); int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); -int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); // STSchema ================================ -void tDestroyTSchema(STSchema *pTSchema); +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); +void tTSchemaDestroy(STSchema *pTSchema); + +// SValue ================================ +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -86,14 +89,26 @@ void tDestroyTSchema(STSchema *pTSchema); #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) -// SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); -void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +// STSRow2 ================================ +#define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) +#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) -// SRowIter ================================ -int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); -void tRowIterClose(SRowIter **ppIter); -SColVal *tRowIterNext(SRowIter *pIter); +int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); +void tTSRowFree(STSRow2 *pRow); +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); +int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); + +// STSRowBuilder ================================ +#define tsRowBuilderInit() ((STSRowBuilder){0}) +#define tsRowBuilderClear(B) \ + do { \ + if ((B)->pBuf) { \ + taosMemoryFree((B)->pBuf); \ + } \ + } while (0) // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); @@ -132,17 +147,29 @@ struct STSchema { int32_t numOfCols; int32_t version; int32_t flen; + int32_t vlen; int32_t tlen; STColumn columns[]; }; -struct SRow { - uint8_t flag; - uint8_t rsv; - uint16_t sver; - uint32_t len; - TSKEY ts; - uint8_t data[]; +#define TSROW_HAS_NONE ((uint8_t)0x1) +#define TSROW_HAS_NULL ((uint8_t)0x2U) +#define TSROW_HAS_VAL ((uint8_t)0x4U) +#define TSROW_KV_SMALL ((uint8_t)0x10U) +#define TSROW_KV_MID ((uint8_t)0x20U) +#define TSROW_KV_BIG ((uint8_t)0x40U) +#pragma pack(push, 1) +struct STSRow2 { + TSKEY ts; + uint8_t flags; + uint8_t data[]; +}; +#pragma pack(pop) + +struct STSRowBuilder { + // STSRow2 tsRow; + int32_t szBuf; + uint8_t *pBuf; }; struct SValue { @@ -231,17 +258,37 @@ typedef struct { int32_t nCols; schema_ver_t version; uint16_t flen; + int32_t vlen; int32_t tlen; STColumn *columns; } STSchemaBuilder; +// use 2 bits for bitmap(default: STSRow/sub block) +#define TD_VTYPE_BITS 2 +#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits +#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder +#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) + +// use 1 bit for bitmap(super block) +#define TD_VTYPE_BITS_I 1 +#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit +#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder +#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) + int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { + if (IS_VAR_DATA_TYPE(type)) { + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + } else { + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; + } +} #endif diff --git a/include/common/trow.h b/include/common/trow.h index 6a71a8844e..9d183c8f80 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -55,14 +55,6 @@ typedef struct STSRow { #define TD_ROW_TP 0x0U // default #define TD_ROW_KV 0x01U -#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits -#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) - -#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit -#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder -#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) - /** * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available @@ -252,7 +244,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal */ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { // The primary TS key is stored separatedly. - return POINTER_SHIFT(TD_ROW_DATA(pRow), flen); + return POINTER_SHIFT(TD_ROW_DATA(pRow), flen - sizeof(TSKEY)); // return POINTER_SHIFT(pRow->ts, flen); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 6a6a5d56d7..90a2adb647 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1275,7 +1275,6 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) nVar++; } } - fLen -= sizeof(TSKEY); int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); @@ -1334,9 +1333,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } + offset += TYPE_BYTES[pColumn->type]; } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1506,7 +1503,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { nVar++; } } - fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1589,9 +1585,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } + + offset += TYPE_BYTES[pColumn->type]; } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1808,7 +1803,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) nVar++; } } - fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1894,9 +1888,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } + + offset += TYPE_BYTES[pColumn->type]; } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3021c586a3..fa63543439 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2055,7 +2055,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB isStartKey = true; tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); - continue; // offset should keep 0 for next column } else if (colDataIsNull_s(pColInfoData, j)) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 728f669fc8..73b887342c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,672 +43,606 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { return code; } -int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) { - int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); - if (code) return code; - - *ppData = pBuffer->pBuf + pBuffer->nBuf; - pBuffer->nBuf += nData; - - return code; -} - // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); -// SRow ======================================================================== -#define KV_FLG_LIT ((uint8_t)0x10) -#define KV_FLG_MID ((uint8_t)0x20) -#define KV_FLG_BIG ((uint8_t)0x30) - -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) - #pragma pack(push, 1) typedef struct { - int16_t nCol; - char idx[]; // uint8_t * | uint16_t * | uint32_t * -} SKVIdx; + int16_t nCols; + uint8_t idx[]; +} STSKVRow; #pragma pack(pop) -#define ROW_SET_BITMAP(PB, FLAG, IDX, VAL) \ - do { \ - if (PB) { \ - switch (FLAG) { \ - case (HAS_NULL | HAS_NONE): \ - SET_BIT1(PB, IDX, VAL); \ - break; \ - case (HAS_VALUE | HAS_NONE): \ - SET_BIT1(PB, IDX, (VAL) ? (VAL)-1 : 0); \ - break; \ - case (HAS_VALUE | HAS_NULL): \ - SET_BIT1(PB, IDX, (VAL)-1); \ - break; \ - case (HAS_VALUE | HAS_NULL | HAS_NONE): \ - SET_BIT2(PB, IDX, VAL); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ - } \ +#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) + +// SValue +static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { + if (IS_VAR_DATA_TYPE(type)) { + return tPutBinary(p, pValue->pData, pValue->nData); + } else { + if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); + return tDataTypes[type].bytes; + } +} + +// STSRow2 ======================================================================== +static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { + if (pb) { + switch (flags & 0xf) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NONE: + if (v) { + SET_BIT1(pb, idx, (uint8_t)1); + } else { + SET_BIT1(pb, idx, (uint8_t)0); + } + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL: + v = v - 1; + SET_BIT1(pb, idx, v); + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + SET_BIT2(pb, idx, v); + break; + + default: + ASSERT(0); + } + } +} +#define SET_IDX(p, i, n, f) \ + do { \ + if ((f)&TSROW_KV_SMALL) { \ + ((uint8_t *)(p))[i] = (n); \ + } else if ((f)&TSROW_KV_MID) { \ + ((uint16_t *)(p))[i] = (n); \ + } else { \ + ((uint32_t *)(p))[i] = (n); \ + } \ } while (0) - -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +#if 0 +int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; +#if 0 + STColumn *pTColumn; + SColVal *pColVal; + int32_t nColVal = taosArrayGetSize(pArray); + int32_t iColVal; - ASSERT(taosArrayGetSize(aColVal) > 0); - ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(nColVal > 0); - // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; - if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - flag |= HAS_VALUE; - maxIdx = nkv; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + // try + uint8_t flags = 0; + uint32_t ntv = 0; + uint32_t nkv = 0; + int16_t nTag = 0; + uint32_t maxIdx = 0; + + iColVal = 0; + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (iColumn == 0) { + ASSERT(pColVal->cid == pTColumn->colId); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + + iColVal++; + } else { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + + if (COL_VAL_IS_NONE(pColVal)) { + flags |= TSROW_HAS_NONE; + } else if (COL_VAL_IS_NULL(pColVal)) { + flags |= TSROW_HAS_NULL; + maxIdx = nkv; + nTag++; + nkv += tPutI16v(NULL, -pTColumn->colId); } else { - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; + flags |= TSROW_HAS_VAL; + maxIdx = nkv; + nTag++; + nkv += tPutI16v(NULL, pTColumn->colId); + nkv += tPutValue(NULL, &pColVal->value, pTColumn->type); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ntv += tPutValue(NULL, &pColVal->value, pTColumn->type); + } } - nIdx++; - } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - flag |= HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { // NULL - flag |= HAS_NULL; - maxIdx = nkv; - nkv += tPutI16v(NULL, -pTColumn->colId); - nIdx++; + } else if (pColVal->cid > pTColumn->colId) { + flags |= TSROW_HAS_NONE; } else { ASSERT(0); } - - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + flags |= TSROW_HAS_NONE; } - } else { // NONE - flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } - // compare --------------- - switch (flag) { - case HAS_NONE: - case HAS_NULL: - ntp = sizeof(SRow); - break; - case HAS_VALUE: - ntp = sizeof(SRow) + ntp; - break; - case (HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - case (HAS_VALUE | HAS_NONE): - case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; - break; - default: - ASSERT(0); - break; - } - if (maxIdx <= UINT8_MAX) { - nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; - flag |= KV_FLG_LIT; - } else if (maxIdx <= UINT16_MAX) { - nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 1) + nkv; - flag |= KV_FLG_MID; + ASSERT(flags); + + // decide + uint32_t nData = 0; + uint32_t nDataT = 0; + uint32_t nDataK = 0; + if (flags == TSROW_HAS_NONE || flags == TSROW_HAS_NULL) { + nData = 0; } else { - nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; - flag |= KV_FLG_BIG; - } - int32_t nRow; - if (nkv < ntp) { - nRow = nkv; - } else { - nRow = ntp; - flag &= ((uint8_t)0x0f); - } - - // alloc -------------- - SRow *pRow = NULL; - code = tBufferReserve(pBuffer, nRow, (void **)&pRow); - if (code) return code; - - // build -------------- - pColVal = (SColVal *)taosArrayGet(aColVal, 0); - - pRow->flag = flag; - pRow->rsv = 0; - pRow->sver = pTSchema->version; - pRow->len = nRow; - memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); - - if (flag == HAS_NONE || flag == HAS_NULL) { - goto _exit; - } - - iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - iTColumn = 1; - pTColumn = pTSchema->columns + iTColumn; - if (flag & 0xf0) { // KV - SKVIdx *pIdx = (SKVIdx *)pRow->data; - int32_t iIdx = 0; - int32_t nv = 0; - uint8_t *pv = NULL; - if (flag & KV_FLG_LIT) { - pv = pIdx->idx + nIdx; - } else if (flag & KV_FLG_MID) { - pv = pIdx->idx + (nIdx << 1); - } else { - pv = pIdx->idx + (nIdx << 2); - } - pIdx->nCol = nIdx; - - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { - if (flag & KV_FLG_LIT) { - ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; - } else if (flag & KV_FLG_MID) { - ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; - } else { - ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; - } - iIdx++; - - nv += tPutI16v(pv + nv, pTColumn->colId); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - nv += tPutU32v(pv + nv, pColVal->value.nData); - memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); - nv += pColVal->value.nData; - } else { - memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); - nv += pTColumn->bytes; - } - } else if (COL_VAL_IS_NULL(pColVal)) { - if (flag & KV_FLG_LIT) { - ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; - } else if (flag & KV_FLG_MID) { - ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; - } else { - ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; - } - iIdx++; - nv += tPutI16v(pv + nv, -pTColumn->colId); - } - - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } - } else { // NONE - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } - } - } else { // TUPLE - uint8_t *pb = NULL; - uint8_t *pf = NULL; - uint8_t *pv = NULL; - int32_t nv = 0; - - switch (flag) { - case (HAS_NULL | HAS_NONE): - pb = pRow->data; + switch (flags) { + case TSROW_HAS_VAL: + nDataT = pTSchema->flen + ntv; break; - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataT = BIT1_SIZE(pTSchema->numOfCols - 1); break; - case (HAS_VALUE | HAS_NONE): - case (HAS_VALUE | HAS_NULL): - pb = pRow->data; - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + nDataT = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - pb = pRow->data; - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - default: - ASSERT(0); - break; - } - - // build impl - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(int32_t *)(pf + pTColumn->offset) = nv; - nv += tPutU32v(pv + nv, pColVal->value.nData); - if (pColVal->value.nData) { - memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); - nv += pColVal->value.nData; - } - } else { - memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); - } - } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - } - - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } - } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } - } - } - -_exit: - return code; -} - -void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - ASSERT(iCol < pTSchema->numOfCols); - ASSERT(pRow->sver == pTSchema->version); - - STColumn *pTColumn = pTSchema->columns + iCol; - - if (iCol == 0) { - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); - return; - } - - if (pRow->flag == HAS_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } - - if (pRow->flag == HAS_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } - - if (pRow->flag & 0xf0) { // KV Row - SKVIdx *pIdx = (SKVIdx *)pRow->data; - uint8_t *pv = NULL; - if (pRow->flag & KV_FLG_LIT) { - pv = pIdx->idx + pIdx->nCol; - } else if (pRow->flag & KV_FLG_MID) { - pv = pIdx->idx + (pIdx->nCol << 1); - } else { - pv = pIdx->idx + (pIdx->nCol << 2); - } - - int16_t lidx = 0; - int16_t ridx = pIdx->nCol - 1; - while (lidx <= ridx) { - int16_t mid = (lidx + ridx) >> 1; - uint8_t *pData = NULL; - if (pRow->flag & KV_FLG_LIT) { - pData = pv + ((uint8_t *)pIdx->idx)[mid]; - } else if (pRow->flag & KV_FLG_MID) { - pData = pv + ((uint16_t *)pIdx->idx)[mid]; - } else { - pData = pv + ((uint32_t *)pIdx->idx)[mid]; - } - - int16_t cid; - pData += tGetI16v(pData, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - } else { - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData > 0) { - pColVal->value.pData = pData; - } else { - pColVal->value.pData = NULL; - } - } else { - memcpy(&pColVal->value.val, pData, pTColumn->bytes); - } - } - return; - } else if (TABS(cid) < pTColumn->colId) { - lidx = mid + 1; - } else { - ridx = mid - 1; - } - } - - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - } else { // Tuple Row - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t bv = ROW_BIT_VALUE; - - switch (pRow->flag) { - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; - break; - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - if (bv) bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pRow->data, iCol - 1); - bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pRow->data, iCol - 1); - pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; break; default: break; - } - - if (bv == ROW_BIT_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } else if (bv == ROW_BIT_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } - - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData) { - pColVal->value.pData = pData; - } else { - pColVal->value.pData = NULL; - } - } else { - memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); - } - } -} - -// SRowIter ======================================== -struct SRowIter { - SRow *pRow; - STSchema *pTSchema; - - int32_t iTColumn; - union { - struct { // kv - int32_t iCol; - SKVIdx *pIdx; - }; - struct { // tuple - uint8_t *pb; - uint8_t *pf; - }; - uint8_t *pv; - }; - - SColVal cv; -}; - -int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { - ASSERT(pRow->sver == pTSchema->version); - - int32_t code = 0; - - SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); - if (pIter == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _exit; - } - - pIter->pRow = pRow; - pIter->pTSchema = pTSchema; - pIter->iTColumn = 0; - - if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; - - if (pRow->flag & 0xf0) { - pIter->iCol = 0; - pIter->pIdx = (SKVIdx *)pRow->data; - if (pRow->flag & KV_FLG_LIT) { - pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; - } else if (pRow->flag & KV_FLG_MID) { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); - } else { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); - } - } else { - switch (pRow->flag) { - case (HAS_NULL | HAS_NONE): - pIter->pb = pRow->data; - break; - case HAS_VALUE: - pIter->pf = pRow->data; - pIter->pv = pIter->pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NONE): - case (HAS_VALUE | HAS_NULL): - pIter->pb = pRow->data; - pIter->pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pIter->pv = pIter->pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - pIter->pb = pRow->data; - pIter->pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pIter->pv = pIter->pf + pTSchema->flen; - break; - default: ASSERT(0); - break; + } + + uint8_t tflags = 0; + if (maxIdx <= UINT8_MAX) { + nDataK = sizeof(STSKVRow) + sizeof(uint8_t) * nTag + nkv; + tflags |= TSROW_KV_SMALL; + } else if (maxIdx <= UINT16_MAX) { + nDataK = sizeof(STSKVRow) + sizeof(uint16_t) * nTag + nkv; + tflags |= TSROW_KV_MID; + } else { + nDataK = sizeof(STSKVRow) + sizeof(uint32_t) * nTag + nkv; + tflags |= TSROW_KV_BIG; + } + + if (nDataT <= nDataK) { + nData = nDataT; + } else { + nData = nDataK; + flags |= tflags; } } -_exit: - if (code) { - *ppIter = NULL; - if (pIter) taosMemoryFree(pIter); - } else { - *ppIter = pIter; - } - return code; -} - -void tRowIterClose(SRowIter **ppIter) { - SRowIter *pIter = *ppIter; - if (pIter) { - taosMemoryFree(pIter); - } - *ppIter = NULL; -} - -SColVal *tRowIterNext(SRowIter *pIter) { - if (pIter->iTColumn >= pIter->pTSchema->numOfCols) { - return NULL; - } - - STColumn *pTColumn = pIter->pTSchema->columns + pIter->iTColumn; - - // timestamp - if (0 == pIter->iTColumn) { - pIter->cv.cid = pTColumn->colId; - pIter->cv.type = pTColumn->type; - pIter->cv.flag = CV_FLAG_VALUE; - memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); - goto _exit; - } - - if (pIter->pRow->flag == HAS_NONE) { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - goto _exit; - } - - if (pIter->pRow->flag == HAS_NULL) { - pIter->cv = COL_VAL_NULL(pTColumn->type, pTColumn->colId); - goto _exit; - } - - if (pIter->pRow->flag & 0xf0) { // KV - if (pIter->iCol < pIter->pIdx->nCol) { - uint8_t *pData; - - if (pIter->pRow->flag & KV_FLG_LIT) { - pData = pIter->pv + ((uint8_t *)pIter->pIdx->idx)[pIter->iCol]; - } else if (pIter->pRow->flag & KV_FLG_MID) { - pData = pIter->pv + ((uint16_t *)pIter->pIdx->idx)[pIter->iCol]; - } else { - pData = pIter->pv + ((uint32_t *)pIter->pIdx->idx)[pIter->iCol]; - } - - int16_t cid; - pData += tGetI16v(pData, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - } else { - pIter->cv.cid = pTColumn->colId; - pIter->cv.type = pTColumn->type; - pIter->cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &pIter->cv.value.nData); - if (pIter->cv.value.nData > 0) { - pIter->cv.value.pData = pData; - } else { - pIter->cv.value.pData = NULL; - } - } else { - memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); - } + // alloc + if (pBuilder) { + // create from a builder + if (nData == 0) { + pBuilder->tsRow.nData = 0; + pBuilder->tsRow.pData = NULL; + } else { + if (pBuilder->szBuf < nData) { + uint8_t *p = taosMemoryRealloc(pBuilder->pBuf, nData); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - - pIter->iCol++; - goto _exit; - } else if (TABS(cid) > pTColumn->colId) { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - goto _exit; - } else { - ASSERT(0); + pBuilder->pBuf = p; + pBuilder->szBuf = nData; } - } else { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + + pBuilder->tsRow.nData = nData; + pBuilder->tsRow.pData = pBuilder->pBuf; + } + + *ppRow = &pBuilder->tsRow; + } else { + // create a new one + *ppRow = (STSRow2 *)taosMemoryMalloc(sizeof(STSRow2)); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; - if (pIter->pb) { - switch (pIter->pRow->flag) { - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); + if (nData == 0) { + (*ppRow)->nData = 0; + (*ppRow)->pData = NULL; + } else { + (*ppRow)->nData = nData; + (*ppRow)->pData = taosMemoryMalloc(nData); + if ((*ppRow)->pData == NULL) { + taosMemoryFree(*ppRow); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + } + + // build + (*ppRow)->flags = flags; + (*ppRow)->sver = pTSchema->version; + + pColVal = (SColVal *)taosArrayGet(pArray, 0); + (*ppRow)->ts = pColVal->value.ts; + + if ((*ppRow)->pData) { + STSKVRow *pTSKVRow = NULL; + uint8_t *pidx = NULL; + uint8_t *pkv = NULL; + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *ptv = NULL; + nkv = 0; + ntv = 0; + iColVal = 1; + + if ((flags & 0xf0) == 0) { + switch (flags & 0xf) { + case TSROW_HAS_VAL: + pf = (*ppRow)->pData; + ptv = pf + pTSchema->flen; break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); - if (bv) bv++; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + pb = (*ppRow)->pData; break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1) + 1; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + pb = (*ppRow)->pData; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + ptv = pf + pTSchema->flen; break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1); + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + pb = (*ppRow)->pData; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + ptv = pf + pTSchema->flen; break; default: ASSERT(0); break; } - - if (bv == ROW_BIT_NONE) { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - goto _exit; - } else if (bv == ROW_BIT_NULL) { - pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - goto _exit; - } - } - - pIter->cv.cid = pTColumn->colId; - pIter->cv.type = pTColumn->type; - pIter->cv.flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); - pData += tGetU32v(pData, &pIter->cv.value.nData); - if (pIter->cv.value.nData > 0) { - pIter->cv.value.pData = pData; - } else { - pIter->cv.value.pData = NULL; - } } else { - memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + pTSKVRow = (STSKVRow *)(*ppRow)->pData; + pTSKVRow->nCols = 0; + pidx = pTSKVRow->idx; + if (flags & TSROW_KV_SMALL) { + pkv = pidx + sizeof(uint8_t) * nTag; + } else if (flags & TSROW_KV_MID) { + pkv = pidx + sizeof(uint16_t) * nTag; + } else { + pkv = pidx + sizeof(uint32_t) * nTag; + } } + + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + + if (COL_VAL_IS_NONE(pColVal)) { + goto _set_none; + } else if (COL_VAL_IS_NULL(pColVal)) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { + goto _set_none; + } else { + ASSERT(0); + } + } else { + goto _set_none; + } + + _set_none: + if ((flags & 0xf0) == 0) { + setBitMap(pb, 0, iColumn - 1, flags); + if (flags & TSROW_HAS_VAL) { // set 0 + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; + } else { + tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); + } + } + } + continue; + + _set_null: + if ((flags & 0xf0) == 0) { + setBitMap(pb, 1, iColumn - 1, flags); + if (flags & TSROW_HAS_VAL) { // set 0 + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; + } else { + tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); + } + } + } else { + SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); + pTSKVRow->nCols++; + nkv += tPutI16v(pkv + nkv, -pTColumn->colId); + } + continue; + + _set_value: + if ((flags & 0xf0) == 0) { + setBitMap(pb, 2, iColumn - 1, flags); + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = ntv; + ntv += tPutValue(ptv + ntv, &pColVal->value, pTColumn->type); + } else { + tPutValue(pf + pTColumn->offset, &pColVal->value, pTColumn->type); + } + } else { + SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); + pTSKVRow->nCols++; + nkv += tPutI16v(pkv + nkv, pColVal->cid); + nkv += tPutValue(pkv + nkv, &pColVal->value, pTColumn->type); + } + continue; + } + } + +#endif +_exit: + return code; +} + +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { + int32_t code = 0; + int32_t rLen = 0; + + TSROW_LEN(pRow, rLen); + (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(*ppRow, pRow, rLen); + +_exit: + return code; +} + +void tTSRowFree(STSRow2 *pRow) { + if (pRow) { + taosMemoryFree(pRow); + } +} + +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + uint8_t isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0; + STColumn *pTColumn = &pTSchema->columns[iCol]; + uint8_t flags = pRow->flags & (uint8_t)0xf; + SValue value; + + ASSERT(iCol < pTSchema->numOfCols); + ASSERT(flags); + ASSERT(pRow->sver == pTSchema->version); + + if (iCol == 0) { + value.ts = pRow->ts; + goto _return_value; + } + + if (flags == TSROW_HAS_NONE) { + goto _return_none; + } else if (flags == TSROW_HAS_NULL) { + goto _return_null; + } + + ASSERT(pRow->nData && pRow->pData); + + if (isTuple) { + uint8_t *pb = pRow->pData; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t *p; + uint8_t b; + + // bit + switch (flags) { + case TSROW_HAS_VAL: + pf = pb; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + goto _return_null; + } + case TSROW_HAS_VAL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_null; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT2(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else if (b == 1) { + goto _return_null; + } else { + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + break; + } + default: + ASSERT(0); + } + + ASSERT(pf); + + p = pf + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pv = pf + pTSchema->flen; + p = pv + *(VarDataOffsetT *)p; + } + tGetValue(p, &value, pTColumn->type); + goto _return_value; + } else { + STSKVRow *pRowK = (STSKVRow *)pRow->pData; + int16_t lidx = 0; + int16_t ridx = pRowK->nCols - 1; + uint8_t *p; + int16_t midx; + uint32_t n; + int16_t cid; + + ASSERT(pRowK->nCols > 0); + + if (pRow->flags & TSROW_KV_SMALL) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_MID) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_BIG) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; + } else { + ASSERT(0); + } + while (lidx <= ridx) { + midx = (lidx + ridx) / 2; + + if (pRow->flags & TSROW_KV_SMALL) { + n = ((uint8_t *)pRowK->idx)[midx]; + } else if (pRow->flags & TSROW_KV_MID) { + n = ((uint16_t *)pRowK->idx)[midx]; + } else { + n = ((uint32_t *)pRowK->idx)[midx]; + } + + n += tGetI16v(p + n, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + goto _return_null; + } else { + n += tGetValue(p + n, &value, pTColumn->type); + goto _return_value; + } + + return; + } else if (TABS(cid) > pTColumn->colId) { + ridx = midx - 1; + } else { + lidx = midx + 1; + } + } + + // not found, return NONE + goto _return_none; + } + +_return_none: + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + +_return_null: + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + +_return_value: + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); + return; +} + +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { + int32_t code = 0; + SColVal cv; + + (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (*ppArray == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + tTSRowGet(pRow, pTSchema, iColumn, &cv); + taosArrayPush(*ppArray, &cv); + } + + _exit: - pIter->iTColumn++; - return &pIter->cv; + return code; +} +#endif +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { + int32_t n = 0; + + TSROW_LEN(pRow, n); + if (p) { + memcpy(p, pRow, n); + } + + return n; +} + +int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { + int32_t n = 0; + + *ppRow = (STSRow2 *)p; + TSROW_LEN(*ppRow, n); + + return n; } // STSchema ======================================== +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { + *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); + if (*ppTSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppTSchema)->numOfCols = ncols; + (*ppTSchema)->version = sver; + (*ppTSchema)->flen = 0; + (*ppTSchema)->vlen = 0; + (*ppTSchema)->tlen = 0; + + for (int32_t iCol = 0; iCol < ncols; iCol++) { + SSchema *pColumn = &pSchema[iCol]; + STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); + + pTColumn->colId = pColumn->colId; + pTColumn->type = pColumn->type; + pTColumn->flags = pColumn->flags; + pTColumn->bytes = pColumn->bytes; + pTColumn->offset = (*ppTSchema)->flen; + + // skip first column + if (iCol) { + (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; + if (IS_VAR_DATA_TYPE(pColumn->type)) { + (*ppTSchema)->vlen += (pColumn->bytes + 5); + } + } + } + + return 0; +} + +void tTSchemaDestroy(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1108,6 +1042,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->nCols = 0; pBuilder->tlen = 0; pBuilder->flen = 0; + pBuilder->vlen = 0; pBuilder->version = version; } @@ -1126,21 +1061,24 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, c pCol->colId = colId; pCol->flags = flags; if (pBuilder->nCols == 0) { - pCol->offset = -1; + pCol->offset = 0; } else { - pCol->offset = pBuilder->flen; - pBuilder->flen += TYPE_BYTES[type]; + STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); + pCol->offset = pTCol->offset + TYPE_BYTES[pTCol->type]; } if (IS_VAR_DATA_TYPE(type)) { pCol->bytes = bytes; pBuilder->tlen += (TYPE_BYTES[type] + bytes); + pBuilder->vlen += bytes - sizeof(VarDataLenT); } else { pCol->bytes = TYPE_BYTES[type]; pBuilder->tlen += TYPE_BYTES[type]; + pBuilder->vlen += TYPE_BYTES[type]; } pBuilder->nCols++; + pBuilder->flen += TYPE_BYTES[type]; ASSERT(pCol->offset < pBuilder->flen); @@ -1159,6 +1097,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { pSchema->numOfCols = pBuilder->nCols; pSchema->tlen = pBuilder->tlen; pSchema->flen = pBuilder->flen; + pSchema->vlen = pBuilder->vlen; #ifdef TD_SUPPORT_BITMAP pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); @@ -1171,43 +1110,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { #endif -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { - STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); - if (pTSchema == NULL) return NULL; - - pTSchema->numOfCols = numOfCols; - pTSchema->version = version; - - // timestamp column - ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); - pTSchema->columns[0].colId = aSchema[0].colId; - pTSchema->columns[0].type = aSchema[0].type; - pTSchema->columns[0].flags = aSchema[0].flags; - pTSchema->columns[0].bytes = aSchema[0].bytes; - pTSchema->columns[0].offset = -1; - - // other columns - for (int32_t iCol = 1; iCol < numOfCols; iCol++) { - SSchema *pSchema = &aSchema[iCol]; - STColumn *pTColumn = &pTSchema->columns[iCol]; - - pTColumn->colId = pSchema->colId; - pTColumn->type = pSchema->type; - pTColumn->flags = pSchema->flags; - pTColumn->bytes = pSchema->bytes; - pTColumn->offset = pTSchema->flen; - - pTSchema->flen += TYPE_BYTES[pTColumn->type]; - } - - return pTSchema; -} - -void tDestroyTSchema(STSchema *pTSchema) { - if (pTSchema) taosMemoryFree(pTSchema); -} - // SColData ======================================== void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; @@ -1594,7 +1496,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); + tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); } *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 52ebd7f879..653cd34b37 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -192,7 +192,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset, colIdx); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); return true; } @@ -217,7 +217,7 @@ bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC return false; } } - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); @@ -244,7 +244,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { @@ -469,7 +469,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell #ifdef TD_SUPPORT_BITMAP colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); #endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset, colIdx - 1); + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); } else if (TD_IS_KV_ROW(pRow)) { SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), compareKvRowColId, TD_EQ); @@ -757,10 +757,11 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if (colIdx < 1) { + if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + offset -= sizeof(TSKEY); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -852,7 +853,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen + pBuilder->nBitmaps; + len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); break; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 7b5d0a8805..8c5d44b8d5 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void *getDataMin(int32_t type, void *value) { +FORCE_INLINE void *getDataMin(int32_t type, void* value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMin; @@ -77,7 +77,7 @@ FORCE_INLINE void *getDataMin(int32_t type, void *value) { return value; } -FORCE_INLINE void *getDataMax(int32_t type, void *value) { +FORCE_INLINE void *getDataMax(int32_t type, void* value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMax; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d335e84c4c..c6e2842c32 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -56,7 +56,7 @@ typedef struct SDataFWriter SDataFWriter; typedef struct SDataFReader SDataFReader; typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; -typedef struct STSDBRowIter STSDBRowIter; +typedef struct SRowIter SRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; @@ -111,9 +111,9 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); -// STSDBRowIter -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); -SColVal *tsdbRowIterNext(STSDBRowIter *pIter); +// SRowIter +void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +SColVal *tRowIterNext(SRowIter *pIter); // SRowMerger int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); @@ -562,7 +562,7 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; -struct STSDBRowIter { +struct SRowIter { TSDBROW *pRow; STSchema *pTSchema; SColVal colVal; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index cf9ae30ff0..65a46331aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -341,7 +341,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->suid = suid; pSkmInfo->uid = uid; - tDestroyTSchema(pSkmInfo->pTSchema); + tTSchemaDestroy(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -365,7 +365,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.suid = suid; pCommitter->skmRow.uid = uid; - tDestroyTSchema(pCommitter->skmRow.pTSchema); + tTSchemaDestroy(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -498,7 +498,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { #if 0 ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); #endif - + pCommitter->nextKey = TSKEY_MAX; // Reader @@ -623,8 +623,7 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -667,8 +666,7 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -708,8 +706,7 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -922,8 +919,8 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { #else tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); #endif - tDestroyTSchema(pCommitter->skmTable.pTSchema); - tDestroyTSchema(pCommitter->skmRow.pTSchema); + tTSchemaDestroy(pCommitter->skmTable.pTSchema); + tTSchemaDestroy(pCommitter->skmRow.pTSchema); } static int32_t tsdbCommitData(SCommitter *pCommitter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b46a003638..9f59707ddc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -595,21 +595,21 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; - STSDBRowIter iter = {0}; - tsdbRowIterInit(&iter, pRow, pTSchema); + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); - SColVal *pColVal = tsdbRowIterNext(&iter); + SColVal *pColVal = tRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); while (pColVal && pColVal->cid < pDCBuilder->cid) { - pColVal = tsdbRowIterNext(&iter); + pColVal = tRowIterNext(&iter); } if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; - pColVal = tsdbRowIterNext(&iter); + pColVal = tRowIterNext(&iter); } else { code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); if (code) return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f4bdeeb387..ec89bed17a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -555,7 +555,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } tBlockDataDestroy(&pReader->bData, 1); - tDestroyTSchema(pReader->skmTable.pTSchema); + tTSchemaDestroy(pReader->skmTable.pTSchema); // del if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1416,7 +1416,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { taosArrayDestroy(pWriter->dReader.aBlockIdx); tBlockDataDestroy(&pWriter->bData, 1); - tDestroyTSchema(pWriter->skmTable.pTSchema); + tTSchemaDestroy(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 5994285647..b0cf09662c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -579,8 +579,8 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } -// STSDBRowIter ====================================================== -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +// SRowIter ====================================================== +void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { pIter->pRow = pRow; if (pRow->type == 0) { ASSERT(pTSchema); @@ -594,7 +594,7 @@ void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { } } -SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { +SColVal *tRowIterNext(SRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); @@ -1084,11 +1084,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); @@ -1106,11 +1106,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5f8120171f..bc09163753 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -139,8 +139,8 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i if (i > 0) { pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; pColList->cols[i].toffset = pColList->flen; - pColList->flen += TYPE_BYTES[type]; } + pColList->flen += TYPE_BYTES[type]; switch (type) { case TSDB_DATA_TYPE_BINARY: pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); From 335a572d86681411d80d4d749c29f1efb9eaba23 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 10:12:25 +0800 Subject: [PATCH 157/165] Revert "Revert "enh: row optimize2"" --- include/common/tdataformat.h | 103 +- include/common/trow.h | 10 +- source/client/src/clientRawBlockWrite.c | 17 +- source/common/src/tdatablock.c | 1 + source/common/src/tdataformat.c | 1200 +++++++++++--------- source/common/src/trow.c | 13 +- source/common/src/ttypes.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 10 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 10 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 14 +- source/libs/parser/src/parInsertUtil.c | 2 +- 13 files changed, 738 insertions(+), 669 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2eda2f66cc..6855287fb2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,17 +27,17 @@ extern "C" { #endif -typedef struct SBuffer SBuffer; -typedef struct SSchema SSchema; -typedef struct STColumn STColumn; -typedef struct STSchema STSchema; -typedef struct SValue SValue; -typedef struct SColVal SColVal; -typedef struct STSRow2 STSRow2; -typedef struct STSRowBuilder STSRowBuilder; -typedef struct STagVal STagVal; -typedef struct STag STag; -typedef struct SColData SColData; +typedef struct SBuffer SBuffer; +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct SValue SValue; +typedef struct SColVal SColVal; +typedef struct SRow SRow; +typedef struct SRowIter SRowIter; +typedef struct STagVal STagVal; +typedef struct STag STag; +typedef struct SColData SColData; #define HAS_NONE ((uint8_t)0x1) #define HAS_NULL ((uint8_t)0x2) @@ -68,13 +68,10 @@ struct SBuffer { void tBufferDestroy(SBuffer *pBuffer); int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); +int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); // STSchema ================================ -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); -void tTSchemaDestroy(STSchema *pTSchema); - -// SValue ================================ -static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); +void tDestroyTSchema(STSchema *pTSchema); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -89,26 +86,14 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) -// STSRow2 ================================ -#define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) -#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) +// SRow ================================ +int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); -int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); -void tTSRowFree(STSRow2 *pRow); -void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); -int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); -int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); - -// STSRowBuilder ================================ -#define tsRowBuilderInit() ((STSRowBuilder){0}) -#define tsRowBuilderClear(B) \ - do { \ - if ((B)->pBuf) { \ - taosMemoryFree((B)->pBuf); \ - } \ - } while (0) +// SRowIter ================================ +int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); +void tRowIterClose(SRowIter **ppIter); +SColVal *tRowIterNext(SRowIter *pIter); // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); @@ -147,29 +132,17 @@ struct STSchema { int32_t numOfCols; int32_t version; int32_t flen; - int32_t vlen; int32_t tlen; STColumn columns[]; }; -#define TSROW_HAS_NONE ((uint8_t)0x1) -#define TSROW_HAS_NULL ((uint8_t)0x2U) -#define TSROW_HAS_VAL ((uint8_t)0x4U) -#define TSROW_KV_SMALL ((uint8_t)0x10U) -#define TSROW_KV_MID ((uint8_t)0x20U) -#define TSROW_KV_BIG ((uint8_t)0x40U) -#pragma pack(push, 1) -struct STSRow2 { - TSKEY ts; - uint8_t flags; - uint8_t data[]; -}; -#pragma pack(pop) - -struct STSRowBuilder { - // STSRow2 tsRow; - int32_t szBuf; - uint8_t *pBuf; +struct SRow { + uint8_t flag; + uint8_t rsv; + uint16_t sver; + uint32_t len; + TSKEY ts; + uint8_t data[]; }; struct SValue { @@ -258,37 +231,17 @@ typedef struct { int32_t nCols; schema_ver_t version; uint16_t flen; - int32_t vlen; int32_t tlen; STColumn *columns; } STSchemaBuilder; -// use 2 bits for bitmap(default: STSRow/sub block) -#define TD_VTYPE_BITS 2 -#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits -#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) - -// use 1 bit for bitmap(super block) -#define TD_VTYPE_BITS_I 1 -#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit -#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder -#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) - int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); -static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); - } else { - memcpy(&pValue->val, p, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); #endif diff --git a/include/common/trow.h b/include/common/trow.h index 9d183c8f80..6a71a8844e 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -55,6 +55,14 @@ typedef struct STSRow { #define TD_ROW_TP 0x0U // default #define TD_ROW_KV 0x01U +#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits +#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder +#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) + +#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit +#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder +#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) + /** * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available @@ -244,7 +252,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal */ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { // The primary TS key is stored separatedly. - return POINTER_SHIFT(TD_ROW_DATA(pRow), flen - sizeof(TSKEY)); + return POINTER_SHIFT(TD_ROW_DATA(pRow), flen); // return POINTER_SHIFT(pRow->ts, flen); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 90a2adb647..6a6a5d56d7 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1275,6 +1275,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) nVar++; } } + fLen -= sizeof(TSKEY); int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); @@ -1333,7 +1334,9 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } } - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1503,6 +1506,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { nVar++; } } + fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1585,8 +1589,9 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1803,6 +1808,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) nVar++; } } + fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1888,8 +1894,9 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index fa63543439..3021c586a3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2055,6 +2055,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB isStartKey = true; tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); + continue; // offset should keep 0 for next column } else if (colDataIsNull_s(pColInfoData, j)) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 73b887342c..728f669fc8 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,606 +43,672 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { return code; } +int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) { + int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); + if (code) return code; + + *ppData = pBuffer->pBuf + pBuffer->nBuf; + pBuffer->nBuf += nData; + + return code; +} + // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); +// SRow ======================================================================== +#define KV_FLG_LIT ((uint8_t)0x10) +#define KV_FLG_MID ((uint8_t)0x20) +#define KV_FLG_BIG ((uint8_t)0x30) + +#define ROW_BIT_NONE ((uint8_t)0x0) +#define ROW_BIT_NULL ((uint8_t)0x1) +#define ROW_BIT_VALUE ((uint8_t)0x2) + #pragma pack(push, 1) typedef struct { - int16_t nCols; - uint8_t idx[]; -} STSKVRow; + int16_t nCol; + char idx[]; // uint8_t * | uint16_t * | uint32_t * +} SKVIdx; #pragma pack(pop) -#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) - -// SValue -static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tPutBinary(p, pValue->pData, pValue->nData); - } else { - if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} - -// STSRow2 ======================================================================== -static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { - if (pb) { - switch (flags & 0xf) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NONE: - if (v) { - SET_BIT1(pb, idx, (uint8_t)1); - } else { - SET_BIT1(pb, idx, (uint8_t)0); - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL: - v = v - 1; - SET_BIT1(pb, idx, v); - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - SET_BIT2(pb, idx, v); - break; - - default: - ASSERT(0); - } - } -} -#define SET_IDX(p, i, n, f) \ - do { \ - if ((f)&TSROW_KV_SMALL) { \ - ((uint8_t *)(p))[i] = (n); \ - } else if ((f)&TSROW_KV_MID) { \ - ((uint16_t *)(p))[i] = (n); \ - } else { \ - ((uint32_t *)(p))[i] = (n); \ - } \ +#define ROW_SET_BITMAP(PB, FLAG, IDX, VAL) \ + do { \ + if (PB) { \ + switch (FLAG) { \ + case (HAS_NULL | HAS_NONE): \ + SET_BIT1(PB, IDX, VAL); \ + break; \ + case (HAS_VALUE | HAS_NONE): \ + SET_BIT1(PB, IDX, (VAL) ? (VAL)-1 : 0); \ + break; \ + case (HAS_VALUE | HAS_NULL): \ + SET_BIT1(PB, IDX, (VAL)-1); \ + break; \ + case (HAS_VALUE | HAS_NULL | HAS_NONE): \ + SET_BIT2(PB, IDX, VAL); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ + } \ } while (0) -#if 0 -int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { + +int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { int32_t code = 0; -#if 0 - STColumn *pTColumn; - SColVal *pColVal; - int32_t nColVal = taosArrayGetSize(pArray); - int32_t iColVal; - ASSERT(nColVal > 0); + ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); - // try - uint8_t flags = 0; - uint32_t ntv = 0; - uint32_t nkv = 0; - int16_t nTag = 0; - uint32_t maxIdx = 0; - - iColVal = 0; - for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - } else { - pColVal = NULL; - } - - if (iColumn == 0) { - ASSERT(pColVal->cid == pTColumn->colId); - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - - iColVal++; - } else { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - - if (COL_VAL_IS_NONE(pColVal)) { - flags |= TSROW_HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - flags |= TSROW_HAS_NULL; - maxIdx = nkv; - nTag++; - nkv += tPutI16v(NULL, -pTColumn->colId); + // scan --------------- + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = taosArrayGetSize(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + int32_t iTColumn = 1; + STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + ntp += TYPE_BYTES[pTColumn->type]; + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + flag |= HAS_VALUE; + maxIdx = nkv; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; } else { - flags |= TSROW_HAS_VAL; - maxIdx = nkv; - nTag++; - nkv += tPutI16v(NULL, pTColumn->colId); - nkv += tPutValue(NULL, &pColVal->value, pTColumn->type); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ntv += tPutValue(NULL, &pColVal->value, pTColumn->type); - } + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; } - } else if (pColVal->cid > pTColumn->colId) { - flags |= TSROW_HAS_NONE; + nIdx++; + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + flag |= HAS_NONE; + } else if (COL_VAL_IS_NULL(pColVal)) { // NULL + flag |= HAS_NULL; + maxIdx = nkv; + nkv += tPutI16v(NULL, -pTColumn->colId); + nIdx++; } else { ASSERT(0); } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + flag |= HAS_NONE; + ntp += TYPE_BYTES[pTColumn->type]; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - flags |= TSROW_HAS_NONE; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + flag |= HAS_NONE; + ntp += TYPE_BYTES[pTColumn->type]; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + + // compare --------------- + switch (flag) { + case HAS_NONE: + case HAS_NULL: + ntp = sizeof(SRow); + break; + case HAS_VALUE: + ntp = sizeof(SRow) + ntp; + break; + case (HAS_NULL | HAS_NONE): + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + break; + default: + ASSERT(0); + break; + } + if (maxIdx <= UINT8_MAX) { + nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; + flag |= KV_FLG_LIT; + } else if (maxIdx <= UINT16_MAX) { + nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 1) + nkv; + flag |= KV_FLG_MID; + } else { + nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; + flag |= KV_FLG_BIG; + } + int32_t nRow; + if (nkv < ntp) { + nRow = nkv; + } else { + nRow = ntp; + flag &= ((uint8_t)0x0f); + } + + // alloc -------------- + SRow *pRow = NULL; + code = tBufferReserve(pBuffer, nRow, (void **)&pRow); + if (code) return code; + + // build -------------- + pColVal = (SColVal *)taosArrayGet(aColVal, 0); + + pRow->flag = flag; + pRow->rsv = 0; + pRow->sver = pTSchema->version; + pRow->len = nRow; + memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); + + if (flag == HAS_NONE || flag == HAS_NULL) { + goto _exit; + } + + iColVal = 1; + pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + iTColumn = 1; + pTColumn = pTSchema->columns + iTColumn; + if (flag & 0xf0) { // KV + SKVIdx *pIdx = (SKVIdx *)pRow->data; + int32_t iIdx = 0; + int32_t nv = 0; + uint8_t *pv = NULL; + if (flag & KV_FLG_LIT) { + pv = pIdx->idx + nIdx; + } else if (flag & KV_FLG_MID) { + pv = pIdx->idx + (nIdx << 1); + } else { + pv = pIdx->idx + (nIdx << 2); + } + pIdx->nCol = nIdx; + + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { + if (flag & KV_FLG_LIT) { + ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; + } else if (flag & KV_FLG_MID) { + ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; + } else { + ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; + } + iIdx++; + + nv += tPutI16v(pv + nv, pTColumn->colId); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + nv += tPutU32v(pv + nv, pColVal->value.nData); + memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); + nv += pColVal->value.nData; + } else { + memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); + nv += pTColumn->bytes; + } + } else if (COL_VAL_IS_NULL(pColVal)) { + if (flag & KV_FLG_LIT) { + ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; + } else if (flag & KV_FLG_MID) { + ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; + } else { + ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; + } + iIdx++; + nv += tPutI16v(pv + nv, -pTColumn->colId); + } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + } else { // TUPLE + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + int32_t nv = 0; + + switch (flag) { + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } + + // build impl + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(int32_t *)(pf + pTColumn->offset) = nv; + nv += tPutU32v(pv + nv, pColVal->value.nData); + if (pColVal->value.nData) { + memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); + nv += pColVal->value.nData; + } + } else { + memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); + } + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + } else { // NULL + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } } - ASSERT(flags); +_exit: + return code; +} - // decide - uint32_t nData = 0; - uint32_t nDataT = 0; - uint32_t nDataK = 0; - if (flags == TSROW_HAS_NONE || flags == TSROW_HAS_NULL) { - nData = 0; - } else { - switch (flags) { - case TSROW_HAS_VAL: - nDataT = pTSchema->flen + ntv; +void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + ASSERT(iCol < pTSchema->numOfCols); + ASSERT(pRow->sver == pTSchema->version); + + STColumn *pTColumn = pTSchema->columns + iCol; + + if (iCol == 0) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + return; + } + + if (pRow->flag == HAS_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } + + if (pRow->flag == HAS_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } + + if (pRow->flag & 0xf0) { // KV Row + SKVIdx *pIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; + if (pRow->flag & KV_FLG_LIT) { + pv = pIdx->idx + pIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pIdx->idx + (pIdx->nCol << 1); + } else { + pv = pIdx->idx + (pIdx->nCol << 2); + } + + int16_t lidx = 0; + int16_t ridx = pIdx->nCol - 1; + while (lidx <= ridx) { + int16_t mid = (lidx + ridx) >> 1; + uint8_t *pData = NULL; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pIdx->idx)[mid]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pIdx->idx)[mid]; + } else { + pData = pv + ((uint32_t *)pIdx->idx)[mid]; + } + + int16_t cid; + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + } else { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData > 0) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pData, pTColumn->bytes); + } + } + return; + } else if (TABS(cid) < pTColumn->colId) { + lidx = mid + 1; + } else { + ridx = mid - 1; + } + } + + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + } else { // Tuple Row + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t bv = ROW_BIT_VALUE; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataT = BIT1_SIZE(pTSchema->numOfCols - 1); + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - nDataT = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; break; default: break; + } + + if (bv == ROW_BIT_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } else if (bv == ROW_BIT_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } + + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); + } + } +} + +// SRowIter ======================================== +struct SRowIter { + SRow *pRow; + STSchema *pTSchema; + + int32_t iTColumn; + union { + struct { // kv + int32_t iCol; + SKVIdx *pIdx; + }; + struct { // tuple + uint8_t *pb; + uint8_t *pf; + }; + uint8_t *pv; + }; + + SColVal cv; +}; + +int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { + ASSERT(pRow->sver == pTSchema->version); + + int32_t code = 0; + + SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); + if (pIter == NULL) { + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _exit; + } + + pIter->pRow = pRow; + pIter->pTSchema = pTSchema; + pIter->iTColumn = 0; + + if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; + + if (pRow->flag & 0xf0) { + pIter->iCol = 0; + pIter->pIdx = (SKVIdx *)pRow->data; + if (pRow->flag & KV_FLG_LIT) { + pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + } else { + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + } + } else { + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + pIter->pb = pRow->data; + break; + case HAS_VALUE: + pIter->pf = pRow->data; + pIter->pv = pIter->pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pIter->pb = pRow->data; + pIter->pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pIter->pv = pIter->pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pIter->pb = pRow->data; + pIter->pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pIter->pv = pIter->pf + pTSchema->flen; + break; + default: ASSERT(0); - } - - uint8_t tflags = 0; - if (maxIdx <= UINT8_MAX) { - nDataK = sizeof(STSKVRow) + sizeof(uint8_t) * nTag + nkv; - tflags |= TSROW_KV_SMALL; - } else if (maxIdx <= UINT16_MAX) { - nDataK = sizeof(STSKVRow) + sizeof(uint16_t) * nTag + nkv; - tflags |= TSROW_KV_MID; - } else { - nDataK = sizeof(STSKVRow) + sizeof(uint32_t) * nTag + nkv; - tflags |= TSROW_KV_BIG; - } - - if (nDataT <= nDataK) { - nData = nDataT; - } else { - nData = nDataK; - flags |= tflags; + break; } } - // alloc - if (pBuilder) { - // create from a builder - if (nData == 0) { - pBuilder->tsRow.nData = 0; - pBuilder->tsRow.pData = NULL; - } else { - if (pBuilder->szBuf < nData) { - uint8_t *p = taosMemoryRealloc(pBuilder->pBuf, nData); - if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - pBuilder->pBuf = p; - pBuilder->szBuf = nData; +_exit: + if (code) { + *ppIter = NULL; + if (pIter) taosMemoryFree(pIter); + } else { + *ppIter = pIter; + } + return code; +} + +void tRowIterClose(SRowIter **ppIter) { + SRowIter *pIter = *ppIter; + if (pIter) { + taosMemoryFree(pIter); + } + *ppIter = NULL; +} + +SColVal *tRowIterNext(SRowIter *pIter) { + if (pIter->iTColumn >= pIter->pTSchema->numOfCols) { + return NULL; + } + + STColumn *pTColumn = pIter->pTSchema->columns + pIter->iTColumn; + + // timestamp + if (0 == pIter->iTColumn) { + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + goto _exit; + } + + if (pIter->pRow->flag == HAS_NONE) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } + + if (pIter->pRow->flag == HAS_NULL) { + pIter->cv = COL_VAL_NULL(pTColumn->type, pTColumn->colId); + goto _exit; + } + + if (pIter->pRow->flag & 0xf0) { // KV + if (pIter->iCol < pIter->pIdx->nCol) { + uint8_t *pData; + + if (pIter->pRow->flag & KV_FLG_LIT) { + pData = pIter->pv + ((uint8_t *)pIter->pIdx->idx)[pIter->iCol]; + } else if (pIter->pRow->flag & KV_FLG_MID) { + pData = pIter->pv + ((uint16_t *)pIter->pIdx->idx)[pIter->iCol]; + } else { + pData = pIter->pv + ((uint32_t *)pIter->pIdx->idx)[pIter->iCol]; } - pBuilder->tsRow.nData = nData; - pBuilder->tsRow.pData = pBuilder->pBuf; - } + int16_t cid; + pData += tGetI16v(pData, &cid); - *ppRow = &pBuilder->tsRow; - } else { - // create a new one - *ppRow = (STSRow2 *)taosMemoryMalloc(sizeof(STSRow2)); - if (*ppRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + } else { + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &pIter->cv.value.nData); + if (pIter->cv.value.nData > 0) { + pIter->cv.value.pData = pData; + } else { + pIter->cv.value.pData = NULL; + } + } else { + memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + } + } + + pIter->iCol++; + goto _exit; + } else if (TABS(cid) > pTColumn->colId) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } else { + ASSERT(0); + } + } else { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; } - if (nData == 0) { - (*ppRow)->nData = 0; - (*ppRow)->pData = NULL; - } else { - (*ppRow)->nData = nData; - (*ppRow)->pData = taosMemoryMalloc(nData); - if ((*ppRow)->pData == NULL) { - taosMemoryFree(*ppRow); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - } - - // build - (*ppRow)->flags = flags; - (*ppRow)->sver = pTSchema->version; - - pColVal = (SColVal *)taosArrayGet(pArray, 0); - (*ppRow)->ts = pColVal->value.ts; - - if ((*ppRow)->pData) { - STSKVRow *pTSKVRow = NULL; - uint8_t *pidx = NULL; - uint8_t *pkv = NULL; - uint8_t *pb = NULL; - uint8_t *pf = NULL; - uint8_t *ptv = NULL; - nkv = 0; - ntv = 0; - iColVal = 1; - - if ((flags & 0xf0) == 0) { - switch (flags & 0xf) { - case TSROW_HAS_VAL: - pf = (*ppRow)->pData; - ptv = pf + pTSchema->flen; + } else { // Tuple + uint8_t bv = ROW_BIT_VALUE; + if (pIter->pb) { + switch (pIter->pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - pb = (*ppRow)->pData; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); + if (bv) bv++; break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - pb = (*ppRow)->pData; - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - ptv = pf + pTSchema->flen; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1) + 1; break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - pb = (*ppRow)->pData; - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - ptv = pf + pTSchema->flen; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1); break; default: ASSERT(0); break; } - } else { - pTSKVRow = (STSKVRow *)(*ppRow)->pData; - pTSKVRow->nCols = 0; - pidx = pTSKVRow->idx; - if (flags & TSROW_KV_SMALL) { - pkv = pidx + sizeof(uint8_t) * nTag; - } else if (flags & TSROW_KV_MID) { - pkv = pidx + sizeof(uint16_t) * nTag; - } else { - pkv = pidx + sizeof(uint32_t) * nTag; + + if (bv == ROW_BIT_NONE) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } else if (bv == ROW_BIT_NULL) { + pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + goto _exit; } } - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - } else { - pColVal = NULL; - } - - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - - if (COL_VAL_IS_NONE(pColVal)) { - goto _set_none; - } else if (COL_VAL_IS_NULL(pColVal)) { - goto _set_null; - } else { - goto _set_value; - } - } else if (pColVal->cid > pTColumn->colId) { - goto _set_none; - } else { - ASSERT(0); - } - } else { - goto _set_none; - } - - _set_none: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 0, iColumn - 1, flags); - if (flags & TSROW_HAS_VAL) { // set 0 - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; - } else { - tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); - } - } - } - continue; - - _set_null: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 1, iColumn - 1, flags); - if (flags & TSROW_HAS_VAL) { // set 0 - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; - } else { - tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); - } - } - } else { - SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); - pTSKVRow->nCols++; - nkv += tPutI16v(pkv + nkv, -pTColumn->colId); - } - continue; - - _set_value: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 2, iColumn - 1, flags); - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = ntv; - ntv += tPutValue(ptv + ntv, &pColVal->value, pTColumn->type); - } else { - tPutValue(pf + pTColumn->offset, &pColVal->value, pTColumn->type); - } - } else { - SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); - pTSKVRow->nCols++; - nkv += tPutI16v(pkv + nkv, pColVal->cid); - nkv += tPutValue(pkv + nkv, &pColVal->value, pTColumn->type); - } - continue; - } - } - -#endif -_exit: - return code; -} - -int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { - int32_t code = 0; - int32_t rLen = 0; - - TSROW_LEN(pRow, rLen); - (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen); - if (*ppRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - memcpy(*ppRow, pRow, rLen); - -_exit: - return code; -} - -void tTSRowFree(STSRow2 *pRow) { - if (pRow) { - taosMemoryFree(pRow); - } -} - -void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - uint8_t isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0; - STColumn *pTColumn = &pTSchema->columns[iCol]; - uint8_t flags = pRow->flags & (uint8_t)0xf; - SValue value; - - ASSERT(iCol < pTSchema->numOfCols); - ASSERT(flags); - ASSERT(pRow->sver == pTSchema->version); - - if (iCol == 0) { - value.ts = pRow->ts; - goto _return_value; - } - - if (flags == TSROW_HAS_NONE) { - goto _return_none; - } else if (flags == TSROW_HAS_NULL) { - goto _return_null; - } - - ASSERT(pRow->nData && pRow->pData); - - if (isTuple) { - uint8_t *pb = pRow->pData; - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t *p; - uint8_t b; - - // bit - switch (flags) { - case TSROW_HAS_VAL: - pf = pb; - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else { - goto _return_null; - } - case TSROW_HAS_VAL | TSROW_HAS_NONE: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else { - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_null; - } else { - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - b = GET_BIT2(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else if (b == 1) { - goto _return_null; - } else { - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - break; - } - default: - ASSERT(0); - } - - ASSERT(pf); - - p = pf + pTColumn->offset; + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pv = pf + pTSchema->flen; - p = pv + *(VarDataOffsetT *)p; - } - tGetValue(p, &value, pTColumn->type); - goto _return_value; - } else { - STSKVRow *pRowK = (STSKVRow *)pRow->pData; - int16_t lidx = 0; - int16_t ridx = pRowK->nCols - 1; - uint8_t *p; - int16_t midx; - uint32_t n; - int16_t cid; - - ASSERT(pRowK->nCols > 0); - - if (pRow->flags & TSROW_KV_SMALL) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; - } else if (pRow->flags & TSROW_KV_MID) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; - } else if (pRow->flags & TSROW_KV_BIG) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; + uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); + pData += tGetU32v(pData, &pIter->cv.value.nData); + if (pIter->cv.value.nData > 0) { + pIter->cv.value.pData = pData; + } else { + pIter->cv.value.pData = NULL; + } } else { - ASSERT(0); + memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); } - while (lidx <= ridx) { - midx = (lidx + ridx) / 2; - - if (pRow->flags & TSROW_KV_SMALL) { - n = ((uint8_t *)pRowK->idx)[midx]; - } else if (pRow->flags & TSROW_KV_MID) { - n = ((uint16_t *)pRowK->idx)[midx]; - } else { - n = ((uint32_t *)pRowK->idx)[midx]; - } - - n += tGetI16v(p + n, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - goto _return_null; - } else { - n += tGetValue(p + n, &value, pTColumn->type); - goto _return_value; - } - - return; - } else if (TABS(cid) > pTColumn->colId) { - ridx = midx - 1; - } else { - lidx = midx + 1; - } - } - - // not found, return NONE - goto _return_none; - } - -_return_none: - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - -_return_null: - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - -_return_value: - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); - return; -} - -int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { - int32_t code = 0; - SColVal cv; - - (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); - if (*ppArray == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { - tTSRowGet(pRow, pTSchema, iColumn, &cv); - taosArrayPush(*ppArray, &cv); - } - - _exit: - return code; -} -#endif -int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { - int32_t n = 0; - - TSROW_LEN(pRow, n); - if (p) { - memcpy(p, pRow, n); - } - - return n; -} - -int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { - int32_t n = 0; - - *ppRow = (STSRow2 *)p; - TSROW_LEN(*ppRow, n); - - return n; + pIter->iTColumn++; + return &pIter->cv; } // STSchema ======================================== -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { - *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); - if (*ppTSchema == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - (*ppTSchema)->numOfCols = ncols; - (*ppTSchema)->version = sver; - (*ppTSchema)->flen = 0; - (*ppTSchema)->vlen = 0; - (*ppTSchema)->tlen = 0; - - for (int32_t iCol = 0; iCol < ncols; iCol++) { - SSchema *pColumn = &pSchema[iCol]; - STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); - - pTColumn->colId = pColumn->colId; - pTColumn->type = pColumn->type; - pTColumn->flags = pColumn->flags; - pTColumn->bytes = pColumn->bytes; - pTColumn->offset = (*ppTSchema)->flen; - - // skip first column - if (iCol) { - (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; - if (IS_VAR_DATA_TYPE(pColumn->type)) { - (*ppTSchema)->vlen += (pColumn->bytes + 5); - } - } - } - - return 0; -} - -void tTSchemaDestroy(STSchema *pTSchema) { - if (pTSchema) taosMemoryFree(pTSchema); -} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1042,7 +1108,6 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->nCols = 0; pBuilder->tlen = 0; pBuilder->flen = 0; - pBuilder->vlen = 0; pBuilder->version = version; } @@ -1061,24 +1126,21 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, c pCol->colId = colId; pCol->flags = flags; if (pBuilder->nCols == 0) { - pCol->offset = 0; + pCol->offset = -1; } else { - STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); - pCol->offset = pTCol->offset + TYPE_BYTES[pTCol->type]; + pCol->offset = pBuilder->flen; + pBuilder->flen += TYPE_BYTES[type]; } if (IS_VAR_DATA_TYPE(type)) { pCol->bytes = bytes; pBuilder->tlen += (TYPE_BYTES[type] + bytes); - pBuilder->vlen += bytes - sizeof(VarDataLenT); } else { pCol->bytes = TYPE_BYTES[type]; pBuilder->tlen += TYPE_BYTES[type]; - pBuilder->vlen += TYPE_BYTES[type]; } pBuilder->nCols++; - pBuilder->flen += TYPE_BYTES[type]; ASSERT(pCol->offset < pBuilder->flen); @@ -1097,7 +1159,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { pSchema->numOfCols = pBuilder->nCols; pSchema->tlen = pBuilder->tlen; pSchema->flen = pBuilder->flen; - pSchema->vlen = pBuilder->vlen; #ifdef TD_SUPPORT_BITMAP pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); @@ -1110,6 +1171,43 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { #endif +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { + STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); + if (pTSchema == NULL) return NULL; + + pTSchema->numOfCols = numOfCols; + pTSchema->version = version; + + // timestamp column + ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + pTSchema->columns[0].colId = aSchema[0].colId; + pTSchema->columns[0].type = aSchema[0].type; + pTSchema->columns[0].flags = aSchema[0].flags; + pTSchema->columns[0].bytes = aSchema[0].bytes; + pTSchema->columns[0].offset = -1; + + // other columns + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SSchema *pSchema = &aSchema[iCol]; + STColumn *pTColumn = &pTSchema->columns[iCol]; + + pTColumn->colId = pSchema->colId; + pTColumn->type = pSchema->type; + pTColumn->flags = pSchema->flags; + pTColumn->bytes = pSchema->bytes; + pTColumn->offset = pTSchema->flen; + + pTSchema->flen += TYPE_BYTES[pTColumn->type]; + } + + return pTSchema; +} + +void tDestroyTSchema(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} + // SColData ======================================== void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; @@ -1496,7 +1594,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); + memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 653cd34b37..52ebd7f879 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -192,7 +192,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset, colIdx); return true; } @@ -217,7 +217,7 @@ bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC return false; } } - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); @@ -244,7 +244,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { @@ -469,7 +469,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell #ifdef TD_SUPPORT_BITMAP colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); #endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset, colIdx - 1); } else if (TD_IS_KV_ROW(pRow)) { SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), compareKvRowColId, TD_EQ); @@ -757,11 +757,10 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { + if (colIdx < 1) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - offset -= sizeof(TSKEY); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -853,7 +852,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; + len = TD_ROW_HEAD_LEN + pBuilder->flen + pBuilder->nBitmaps; TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); break; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 8c5d44b8d5..7b5d0a8805 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void *getDataMin(int32_t type, void* value) { +FORCE_INLINE void *getDataMin(int32_t type, void *value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMin; @@ -77,7 +77,7 @@ FORCE_INLINE void *getDataMin(int32_t type, void* value) { return value; } -FORCE_INLINE void *getDataMax(int32_t type, void* value) { +FORCE_INLINE void *getDataMax(int32_t type, void *value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMax; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c6e2842c32..d335e84c4c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -56,7 +56,7 @@ typedef struct SDataFWriter SDataFWriter; typedef struct SDataFReader SDataFReader; typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; +typedef struct STSDBRowIter STSDBRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; @@ -111,9 +111,9 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); -// SRowIter -void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); -SColVal *tRowIterNext(SRowIter *pIter); +// STSDBRowIter +void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); @@ -562,7 +562,7 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; -struct SRowIter { +struct STSDBRowIter { TSDBROW *pRow; STSchema *pTSchema; SColVal colVal; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 65a46331aa..cf9ae30ff0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -341,7 +341,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->suid = suid; pSkmInfo->uid = uid; - tTSchemaDestroy(pSkmInfo->pTSchema); + tDestroyTSchema(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -365,7 +365,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.suid = suid; pCommitter->skmRow.uid = uid; - tTSchemaDestroy(pCommitter->skmRow.pTSchema); + tDestroyTSchema(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -498,7 +498,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { #if 0 ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); #endif - + pCommitter->nextKey = TSKEY_MAX; // Reader @@ -623,7 +623,8 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -666,7 +667,8 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -706,7 +708,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -919,8 +922,8 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { #else tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); #endif - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - tTSchemaDestroy(pCommitter->skmRow.pTSchema); + tDestroyTSchema(pCommitter->skmTable.pTSchema); + tDestroyTSchema(pCommitter->skmRow.pTSchema); } static int32_t tsdbCommitData(SCommitter *pCommitter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9f59707ddc..b46a003638 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -595,21 +595,21 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; - SRowIter iter = {0}; - tRowIterInit(&iter, pRow, pTSchema); + STSDBRowIter iter = {0}; + tsdbRowIterInit(&iter, pRow, pTSchema); - SColVal *pColVal = tRowIterNext(&iter); + SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); while (pColVal && pColVal->cid < pDCBuilder->cid) { - pColVal = tRowIterNext(&iter); + pColVal = tsdbRowIterNext(&iter); } if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; - pColVal = tRowIterNext(&iter); + pColVal = tsdbRowIterNext(&iter); } else { code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); if (code) return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ec89bed17a..f4bdeeb387 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -555,7 +555,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } tBlockDataDestroy(&pReader->bData, 1); - tTSchemaDestroy(pReader->skmTable.pTSchema); + tDestroyTSchema(pReader->skmTable.pTSchema); // del if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1416,7 +1416,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { taosArrayDestroy(pWriter->dReader.aBlockIdx); tBlockDataDestroy(&pWriter->bData, 1); - tTSchemaDestroy(pWriter->skmTable.pTSchema); + tDestroyTSchema(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index b0cf09662c..5994285647 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -579,8 +579,8 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } -// SRowIter ====================================================== -void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +// STSDBRowIter ====================================================== +void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { pIter->pRow = pRow; if (pRow->type == 0) { ASSERT(pTSchema); @@ -594,7 +594,7 @@ void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { } } -SColVal *tRowIterNext(SRowIter *pIter) { +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); @@ -1084,11 +1084,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); @@ -1106,11 +1106,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index bc09163753..5f8120171f 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -139,8 +139,8 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i if (i > 0) { pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; pColList->cols[i].toffset = pColList->flen; + pColList->flen += TYPE_BYTES[type]; } - pColList->flen += TYPE_BYTES[type]; switch (type) { case TSDB_DATA_TYPE_BINARY: pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); From b27faba7efb06f9789c240be1285e15d4453af8a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 23 Nov 2022 17:07:45 +0800 Subject: [PATCH 158/165] fix:initialize maxts --- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 15 +++-- tests/script/tsim/stream/state0.sim | 60 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index de2a7b9dac..8db450ad50 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1849,12 +1849,12 @@ FETCH_NEXT_BLOCK: prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; printDataBlock(pDelBlock, "stream scan delete data"); if (pInfo->tqReader) { blockDataDestroy(pDelBlock); } if (pInfo->pDeleteDataRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; return pInfo->pDeleteDataRes; } else { goto FETCH_NEXT_BLOCK; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index dd02ce9cd4..7fd2fcb5b8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3364,22 +3364,23 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t waterMark, uint16_t type, - int32_t tsColIndex) { +void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, waterMark, type, tsColIndex); + initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup); return; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); + pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } + pScanInfo->twAggSup = *pTwSup; } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, @@ -4102,8 +4103,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, - pInfo->primaryTsIndex); + initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); } return pOperator; @@ -4606,8 +4606,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); - initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, - pInfo->primaryTsIndex); + initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 87d7d4b7fc..7c922658c9 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -731,5 +731,65 @@ if $data32 != 1 then goto loop9 endi +sql drop stream if exists streams5; +sql drop database if exists test5; +sql create database test5; +sql use test5; +sql create table tb (ts timestamp, a int); +sql insert into tb values (now + 1m , 1 ); +sql create table b (c timestamp, d int, e int , f int, g double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); +sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); +sql select * from streamt order by c1, c2, c3; + +print data00:$data00 +print data01:$data01 + +sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); +sql select * from streamt order by c1, c2, c3; + +print data00:$data00 +print data01:$data01 + +sql insert into b values(1648791213001,1,2,2,2.0); +sql insert into b values(1648791213002,1,3,3,3.0); +sql insert into tb values(1648791213003,1); + +sql select * from streamt; +print data00:$data00 +print data01:$data01 + +sql delete from b where c >= 1648791213001 and c <= 1648791213002; +sql insert into b values(1648791223003,2,2,3,1.0); insert into b values(1648791223002,2,3,3,3.0); +sql insert into tb values (now + 1m , 1 ); + +sql select * from streamt; +print data00:$data00 +print data01:$data01 + +sql insert into b(c,d) values (now + 6m , 6 ); +sql delete from b where c >= 1648791213001 and c <= 1648791233005;; + +$loop_count = 0 +loop10: + +sleep 200 + +sql select c2 from streamt; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop10 +endi + +if $data00 != 2 then + print =====data00=$data00 + goto loop10 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From b460974f43d248edceee368e371d043136bbbdca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Nov 2022 10:32:31 +0800 Subject: [PATCH 159/165] fix(query): fix syntax error. --- source/libs/executor/src/exchangeoperator.c | 27 ++++++++------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 04caccbf8f..2c79b77e16 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -75,11 +75,9 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } while (1) { - int32_t v = 0; - sem_getvalue(&pExchangeInfo->ready, &v); - qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, (void*)pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); - + qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo)); tsem_wait(&pExchangeInfo->ready); + if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -372,9 +370,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t index = pWrapper->sourceIndex; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - int32_t v = 0, v1 = 0; - sem_getvalue(&pExchangeInfo->ready, &v); - if (code == TSDB_CODE_SUCCESS) { pSourceDataInfo->pRsp = pMsg->pData; @@ -386,22 +381,23 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, v, (void*)pExchangeInfo->ready.__align, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, tstrerror(code), v, - (void*) pExchangeInfo->ready.__align, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; - code = tsem_post(&pExchangeInfo->ready); - ASSERT(code == TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + code = TAOS_SYSTEM_ERROR(code); + qError("failed to invoke post when fetch rsp is ready, code:%s, %p", tstrerror(code), pExchangeInfo); + } taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - return TSDB_CODE_SUCCESS; + return code; } int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { @@ -566,9 +562,6 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; - int32_t value = 0; - sem_getvalue(&pExchangeInfo->ready, &value); - tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); From cd46ca0f5a3a5cb9e3fbb80f8a12724bd6d91787 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 24 Nov 2022 11:12:08 +0800 Subject: [PATCH 160/165] fix(shell): add help command and fix crete table filed complete include binary --- tools/shell/inc/shellAuto.h | 3 +++ tools/shell/src/shellAuto.c | 43 +++++++++++++++++++++++------------ tools/shell/src/shellEngine.c | 6 +++++ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tools/shell/inc/shellAuto.h b/tools/shell/inc/shellAuto.h index f86090d618..b7bf5fa101 100644 --- a/tools/shell/inc/shellAuto.h +++ b/tools/shell/inc/shellAuto.h @@ -39,4 +39,7 @@ void callbackAutoTab(char* sqlstr, TAOS* pSql, bool usedb); // introduction void printfIntroduction(); +// show all commands help +void showHelp(); + #endif diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 534ecf3c4d..01c8042c0e 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -108,6 +108,7 @@ SWords shellCommands[] = { {"drop topic ;", 0, 0, NULL}, {"drop stream ;", 0, 0, NULL}, {"explain select", 0, 0, NULL}, // 44 append sub sql + {"help;", 0, 0, NULL}, {"grant all on to ;", 0, 0, NULL}, {"grant read on to ;", 0, 0, NULL}, {"grant write on to ;", 0, 0, NULL}, @@ -386,6 +387,8 @@ void showHelp() { drop stream ;\n\ ----- E ----- \n\ explain select clause ...\n\ + ----- H ----- \n\ + help;\n\ ----- I ----- \n\ insert into values(...) ;\n\ insert into using tags(...) values(...) ;\n\ @@ -1478,24 +1481,36 @@ bool matchSelectQuery(TAOS* con, SShellCmd* cmd) { // if is input create fields or tags area, return true bool isCreateFieldsArea(char* p) { - char* left = strrchr(p, '('); - if (left == NULL) { - // like 'create table st' - return false; - } + // put to while, support like create table st(ts timestamp, bin1 binary(16), bin2 + blank + TAB + char* p1 = strdup(p); + bool ret = false; + while (1) { + char* left = strrchr(p1, '('); + if (left == NULL) { + // like 'create table st' + ret = false; + break; + } - char* right = strrchr(p, ')'); - if (right == NULL) { - // like 'create table st( ' - return true; - } + char* right = strrchr(p1, ')'); + if (right == NULL) { + // like 'create table st( ' + ret = true; + break; + } - if (left > right) { - // like 'create table st( ts timestamp, age int) tags(area ' - return true; + if (left > right) { + // like 'create table st( ts timestamp, age int) tags(area ' + ret = true; + break; + } + + // set string end by small for next strrchr search + *left = 0; } + taosMemoryFree(p1); - return false; + return ret; } bool matchCreateTable(TAOS* con, SShellCmd* cmd) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8402a5a589..577021f460 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -134,6 +134,12 @@ int32_t shellRunCommand(char *command, bool recordHistory) { return 0; } + // add help or help; + if(strcmp(command, "help") == 0 || strcmp(command, "help;") == 0) { + showHelp(); + return 0; + } + if (recordHistory) shellRecordCommandToHistory(command); char quote = 0, *cmd = command; From a0aa07b41dabde141d880d40b4ebbb7f7afd6ceb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 11:45:40 +0800 Subject: [PATCH 161/165] test: adjust asan case --- tests/parallel_test/cases.task | 6 +++--- tests/system-test/2-query/smaTest.py | 25 ++++++++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 27611a1a17..ee6f0a393c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -417,7 +417,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py -#,,,system-test,python3 ./test.py -f 1-insert/alter_database.py +,,,system-test,python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -561,8 +561,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R -,,,system-test,python3 ./test.py -f 2-query/smaTest.py -,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 0390bae114..04fb893e75 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -44,8 +44,8 @@ class TDTestCase: def run(self): # insert data dbname = "db" - self.insert_data1(f"{dbname}.t1", self.ts, 1000*10000) - self.insert_data1(f"{dbname}.t4", self.ts, 1000*10000) + self.insert_data1(f"{dbname}.t1", self.ts, 10*10000) + self.insert_data1(f"{dbname}.t4", self.ts, 10*10000) # test base case # self.test_case1() tdLog.debug(" LIMIT test_case1 ............ [OK]") @@ -53,7 +53,6 @@ class TDTestCase: # self.test_case2() tdLog.debug(" LIMIT test_case2 ............ [OK]") - # stop def stop(self): tdSql.close() @@ -77,15 +76,17 @@ class TDTestCase: # insert data1 def insert_data(self, tbname, ts_start, count): - pre_insert = "insert into %s values"%tbname + pre_insert = "insert into %s values" % tbname sql = pre_insert - tdLog.debug("insert table %s rows=%d ..."%(tbname, count)) + tdLog.debug("insert table %s rows=%d ..." % (tbname, count)) for i in range(count): - sql += " (%d,%d)"%(ts_start + i*1000, i ) - if i >0 and i%30000 == 0: + sql += " (%d,%d)" % (ts_start + i*1000, i) + if i > 0 and i % 20000 == 0: + tdLog.info("%d rows inserted" % i) tdSql.execute(sql) sql = pre_insert # end sql + tdLog.info("insert_data end") if sql != pre_insert: tdSql.execute(sql) @@ -93,15 +94,17 @@ class TDTestCase: return def insert_data1(self, tbname, ts_start, count): - pre_insert = "insert into %s values"%tbname + pre_insert = "insert into %s values" % tbname sql = pre_insert - tdLog.debug("insert table %s rows=%d ..."%(tbname, count)) + tdLog.debug("insert table %s rows=%d ..." % (tbname, count)) for i in range(count): - sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) - if i >0 and i%30000 == 0: + sql += " (%d,%d,%d)" % (ts_start + i*1000, i, i+1) + if i > 0 and i % 20000 == 0: + tdLog.info("%d rows inserted" % i) tdSql.execute(sql) sql = pre_insert # end sql + tdLog.info("insert_data1 end") if sql != pre_insert: tdSql.execute(sql) From 862a51e6ee0949f4e04d5bcf06a7238b286dd12d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 12:03:05 +0800 Subject: [PATCH 162/165] refactor(sync): add timestamp trace log when elect and heartbeat --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncMain.c | 17 ++++++-- source/libs/sync/src/syncReplication.c | 3 ++ source/libs/sync/src/syncUtil.c | 55 +++++++++++++++++++++----- 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 1e72b4eb26..aa8d3bef51 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -232,7 +232,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, S int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h); -bool syncNodeHeartbeatTimeout(SSyncNode* pSyncNode); +bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index acaf499cbb..7ceec29be4 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -124,6 +124,7 @@ typedef struct SyncHeartbeat { SyncIndex commitIndex; SyncTerm privateTerm; SyncTerm minMatchIndex; + int64_t timeStamp; } SyncHeartbeat; typedef struct SyncHeartbeatReply { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 008e49c20c..7dab496a5b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -640,7 +640,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } // heartbeat timeout - if (syncNodeHeartbeatTimeout(pSyncNode)) { + if (syncNodeHeartbeatReplyTimeout(pSyncNode)) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; sNError(pSyncNode, "failed to sync propose since hearbeat timeout, type:%s, last:%" PRId64 ", cmt:%" PRId64, TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); @@ -2039,6 +2039,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; + pSyncMsg->timeStamp = taosGetTimestampMs(); // send msg syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); @@ -2094,7 +2095,7 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand return code; } -bool syncNodeHeartbeatTimeout(SSyncNode* pSyncNode) { +bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { if (pSyncNode->replicaNum == 1) { return false; } @@ -2148,7 +2149,11 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeat* pMsg = pRpcMsg->pCont; - syncLogRecvHeartbeat(ths, pMsg, ""); + + int64_t tsMs = taosGetTimestampMs(); + char buf[128]; + snprintf(buf, sizeof(buf), "recv local time:%" PRId64, tsMs); + syncLogRecvHeartbeat(ths, pMsg, buf); SRpcMsg rpcMsg = {0}; (void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId); @@ -2161,6 +2166,8 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->timeStamp = taosGetTimestampMs(); if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { + syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); + syncNodeResetElectTimer(ths); ths->minMatchIndex = pMsg->minMatchIndex; @@ -2220,9 +2227,11 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeatReply* pMsg = pRpcMsg->pCont; - syncLogRecvHeartbeatReply(ths, pMsg, ""); int64_t tsMs = taosGetTimestampMs(); + char buf[128]; + snprintf(buf, sizeof(buf), "recv local time:%" PRId64, tsMs); + syncLogRecvHeartbeatReply(ths, pMsg, buf); // update last reply time, make decision whether the other node is alive or not syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index de5f71e5a9..54c29febe5 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -213,9 +213,11 @@ int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcM } int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { + int64_t ts = taosGetTimestampMs(); for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { SRpcMsg rpcMsg = {0}; if (syncBuildHeartbeat(&rpcMsg, pSyncNode->vgId) != 0) { + sError("vgId:%d, build sync-heartbeat error", pSyncNode->vgId); continue; } @@ -226,6 +228,7 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; + pSyncMsg->timeStamp = ts; // send msg syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4c84786286..1e5a268e97 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "syncUtil.h" +#include "syncIndexMgr.h" #include "syncMessage.h" #include "syncRaftCfg.h" #include "syncRaftStore.h" @@ -175,6 +176,36 @@ void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { } } +// for leader +static void syncHearbeatReplyTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { + int32_t len = 5; + + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->replicasId[i])); + + if (i < pSyncNode->replicaNum - 1) { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs); + } else { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs); + } + } +} + +// for follower +static void syncHearbeatTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { + int32_t len = 4; + + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->replicasId[i])); + + if (i < pSyncNode->replicaNum - 1) { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs); + } else { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs); + } + } +} + static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int32_t len = 1; @@ -221,6 +252,12 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo char peerStr[1024] = "{"; syncPeerState2Str(pNode, peerStr, sizeof(peerStr)); + char hbrTimeStr[256] = "hbr:{"; + syncHearbeatReplyTime2Str(pNode, hbrTimeStr, sizeof(hbrTimeStr)); + + char hbTimeStr[256] = "hb:{"; + syncHearbeatTime2Str(pNode, hbTimeStr, sizeof(hbTimeStr)); + int32_t quorum = syncNodeDynamicQuorum(pNode); char eventLog[512]; // {0}; @@ -243,12 +280,13 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 - ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", + ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->pRaftCfg->isStandBy, aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, - pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); + pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, + hbrTimeStr); } } @@ -395,9 +433,8 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 - "}, %s", - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); + "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, %s", + host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s); } void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const char* s) { @@ -406,9 +443,9 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 + "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, %s", - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); + host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s); } void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) { @@ -416,8 +453,8 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, - pMsg->term, pMsg->privateTerm, s); + sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port, + pMsg->term, pMsg->timeStamp, s); } void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) { From 932cec70dfb2ef176be2718735fae13ab6430d34 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 12:06:52 +0800 Subject: [PATCH 163/165] test: add asan case --- tests/parallel_test/cases.task | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 27611a1a17..c61cc35609 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -595,8 +595,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -R -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py @@ -820,7 +820,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 @@ -913,7 +913,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 From 5d5538a12cfb084abbe7609b22a292330b7bfc16 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Nov 2022 12:36:14 +0800 Subject: [PATCH 164/165] fix(query): fix error in seq fetch data block. --- source/libs/executor/src/exchangeoperator.c | 52 ++++++++++++++------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 2c79b77e16..cf6263148a 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -63,6 +63,7 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +static int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo); static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo) { @@ -112,20 +113,12 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn break; } - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - int32_t index = 0; - char* pStart = pRetrieveRsp->data; - while (index++ < pRetrieveRsp->numOfBlocks) { - SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - - taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + code = doExtractResultBlocks(pExchangeInfo, pDataInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; } + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pDataInfo->startTime, pOperator); pDataInfo->totalRows += pRetrieveRsp->numOfRows; @@ -571,10 +564,32 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo) { + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t index = 0; + int32_t code = 0; + while (index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + return code; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + } + + return code; +} + int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t code = 0; size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); int64_t startTs = taosGetTimestampUs(); @@ -614,11 +629,12 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { continue; } + code = doExtractResultBlocks(pExchangeInfo, pDataInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - - char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); - if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, @@ -641,6 +657,10 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { taosMemoryFreeClear(pDataInfo->pRsp); return TSDB_CODE_SUCCESS; } + + _error: + pTaskInfo->code = code; + return code; } int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { From 1d2944973de5221409308890a501f08c8bbb2df3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 13:36:05 +0800 Subject: [PATCH 165/165] test: remove case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4054a0c3eb..64eca5fda6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -418,7 +418,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py